16 changed files with 377 additions and 31 deletions
@ -0,0 +1,175 @@ |
|||
# -*- coding: utf-8 -*- |
|||
|
|||
"""Visitor tracking models.""" |
|||
|
|||
import time |
|||
import requests |
|||
|
|||
import rophako.jsondb as JsonDB |
|||
from rophako.utils import remote_addr, pretty_time, server_name |
|||
|
|||
def track_visit(request, session): |
|||
"""Main logic to track and log visitor details.""" |
|||
|
|||
# Get their tracking cookie value. The value will either be their HTTP |
|||
# referrer (if exists and valid) or else a "1". |
|||
cookie = session.get("tracking") |
|||
addr = remote_addr() |
|||
values = dict() # Returnable traffic values |
|||
|
|||
# Log hit counts. We need four kinds: |
|||
# - Unique today - Unique total |
|||
# - Hits today - Hits total |
|||
today = pretty_time("%Y-%m-%d", time.time()) |
|||
files = { |
|||
"unique/{}".format(today) : "unique_today", |
|||
"unique/total" : "unique_total", |
|||
"hits/{}".format(today) : "hits_today", |
|||
"hits/total" : "hits_total", |
|||
} |
|||
|
|||
# Go through the hit count files. Update them only if their tracking |
|||
# cookie was not present. |
|||
for file, key in files.items(): |
|||
dbfile = "traffic/{}".format(file) |
|||
if file.startswith("hits"): |
|||
# Hit file is just a simple counter. |
|||
db = dict(hits=0) |
|||
if JsonDB.exists(dbfile): |
|||
db = JsonDB.get(dbfile) |
|||
|
|||
# Update it? |
|||
if not cookie: |
|||
db["hits"] += 1 |
|||
JsonDB.commit(dbfile, db) |
|||
|
|||
# Store the copy. |
|||
values[key] = db["hits"] |
|||
else: |
|||
# Unique file is a collection of IP addresses. |
|||
db = dict() |
|||
if JsonDB.exists(dbfile): |
|||
db = JsonDB.get(dbfile) |
|||
|
|||
# Update with their IP? |
|||
if not cookie and not addr in db: |
|||
db[addr] = time.time() |
|||
JsonDB.commit(dbfile, db) |
|||
|
|||
# Store the copy. |
|||
values[key] = len(db.keys()) |
|||
|
|||
# Log their HTTP referrer. |
|||
referrer = "1" |
|||
if request.referrer: |
|||
# Branch and check this. |
|||
referrer = log_referrer(request, request.referrer) |
|||
if not referrer: |
|||
# Wasn't a valid referrer. |
|||
referrer = "1" |
|||
|
|||
# Set their tracking cookie. |
|||
if not cookie: |
|||
cookie = referrer |
|||
session["tracking"] = cookie |
|||
|
|||
return values |
|||
|
|||
|
|||
def log_referrer(request, link): |
|||
"""Double check the referring URL.""" |
|||
|
|||
# Ignore if same domain. |
|||
if link.startswith(request.url_root): |
|||
print "Referrer is same host!" |
|||
return None |
|||
|
|||
# See if the URL really links back to us. |
|||
hostname = server_name() |
|||
r = requests.get(link) |
|||
if hostname in r.text: |
|||
# Log it. |
|||
db = list() |
|||
if JsonDB.exists("traffic/referrers"): |
|||
# Don't cache the result -- the list can get huge! |
|||
db = JsonDB.get("traffic/referrers", cache=False) |
|||
db.append(link) |
|||
JsonDB.commit("traffic/referrers", db, cache=False) |
|||
return link |
|||
|
|||
return None |
|||
|
|||
|
|||
def get_visitor_details(): |
|||
"""Retrieve detailed visitor information for the frontend.""" |
|||
result = { |
|||
"traffic": [], # Historical traffic data |
|||
"most_unique": [ "0000-00-00", 0 ], # Day with the most unique |
|||
"most_hits": [ "0000-00-00", 0 ], # Day with the most hits |
|||
"oldest": None, # Oldest day on record. |
|||
} |
|||
|
|||
# List all the documents. |
|||
hits = JsonDB.list_docs("traffic/hits") |
|||
for date in sorted(hits): |
|||
if date == "total": continue |
|||
if not result["oldest"]: |
|||
result["oldest"] = date |
|||
|
|||
# Get the DBs. |
|||
hits_db = JsonDB.get("traffic/hits/{}".format(date), cache=False) |
|||
uniq_db = JsonDB.get("traffic/unique/{}".format(date), cache=False) |
|||
|
|||
# Most we've seen? |
|||
if hits_db["hits"] > result["most_hits"][1]: |
|||
result["most_hits"] = [ date, hits_db["hits"] ] |
|||
if len(uniq_db.keys()) > result["most_unique"][1]: |
|||
result["most_unique"] = [ date, len(uniq_db.keys()) ] |
|||
|
|||
result["traffic"].append(dict( |
|||
date=date, |
|||
hits=hits_db["hits"], |
|||
unique=len(uniq_db.keys()), |
|||
)) |
|||
|
|||
return result |
|||
|
|||
|
|||
def get_referrers(recent=25): |
|||
"""Retrieve the referrer details. Returns results in this format: |
|||
|
|||
``` |
|||
{ |
|||
referrers: [ |
|||
["http://...", 20], # Pre-sorted by number of hits |
|||
], |
|||
recent: [ recent list ] |
|||
} |
|||
``` |
|||
""" |
|||
db = [] |
|||
if JsonDB.exists("traffic/referrers"): |
|||
db = JsonDB.get("traffic/referrers", cache=False) |
|||
|
|||
# Count the links. |
|||
unique = dict() |
|||
for link in db: |
|||
if not link in unique: |
|||
unique[link] = 1 |
|||
else: |
|||
unique[link] += 1 |
|||
|
|||
# Sort them by popularity. |
|||
result = dict( |
|||
referrers=[], |
|||
recent=[], |
|||
) |
|||
|
|||
sorted_links = sorted(unique.keys(), key=lambda x: unique[x], reverse=True) |
|||
for link in sorted_links: |
|||
result["referrers"].append([ link, unique[link] ]) |
|||
|
|||
recent = 0 - recent |
|||
result["recent"] = db[recent:] |
|||
|
|||
return result |
|||
@ -0,0 +1,43 @@ |
|||
# -*- coding: utf-8 -*- |
|||
|
|||
"""Endpoints for visitor tracking functions.""" |
|||
|
|||
from flask import Blueprint, g |
|||
import re |
|||
|
|||
import rophako.model.tracking as Tracking |
|||
from rophako.utils import template |
|||
|
|||
mod = Blueprint("tracking", __name__, url_prefix="/tracking") |
|||
|
|||
|
|||
@mod.route("/") |
|||
def index(): |
|||
return template("tracking/index.html") |
|||
|
|||
|
|||
@mod.route("/visitors") |
|||
def visitors(): |
|||
g.info["history"] = Tracking.get_visitor_details() |
|||
return template("tracking/visitors.html") |
|||
|
|||
|
|||
@mod.route("/referrers") |
|||
def referrers(): |
|||
g.info["referrers"] = Tracking.get_referrers() |
|||
|
|||
# Filter some of the links. |
|||
for i, link in enumerate(g.info["referrers"]["referrers"]): |
|||
# Clean up useless Google links. |
|||
if "google" in link[0] and re.search(r'/(?:imgres|url|search|translate\w+)?/', link[0]): |
|||
g.info["referrers"]["referrers"][i] = None |
|||
|
|||
# Make the links word-wrap properly. |
|||
filtered = [ |
|||
[ re.sub(r'(.{20})', r'\1<wbr>', x[0]), x[1] ] |
|||
for x in g.info["referrers"]["referrers"] |
|||
if x is not None |
|||
] |
|||
g.info["referrers"]["referrers"] = filtered |
|||
|
|||
return template("tracking/referrers.html") |
|||
@ -0,0 +1,13 @@ |
|||
{% extends "layout.html" %} |
|||
{% block title %}Visitor Tracking{% endblock %} |
|||
|
|||
{% block content %} |
|||
|
|||
<h1>Visitor Tracking</h1> |
|||
|
|||
<ul> |
|||
<li><a href="{{ url_for('tracking.visitors') }}">Unique Visitors & Visits</a></li> |
|||
<li><a href="{{ url_for('tracking.referrers') }}">HTTP Referrers</a></li> |
|||
</ul> |
|||
|
|||
{% endblock %} |
|||
@ -0,0 +1,44 @@ |
|||
{% extends "layout.html" %} |
|||
{% block title %}Referring URLs{% endblock %} |
|||
|
|||
{% block content %} |
|||
|
|||
<h1>Referring URLs</h1> |
|||
|
|||
This table lists the HTTP referrers to this site, in order of popularity. For |
|||
the most recent 25 links, see <a href="#recent">the end of this page</a>.<p> |
|||
|
|||
<div style="height: 450px; overflow: auto"> |
|||
<table class="table" width="100%"> |
|||
<thead> |
|||
<tr> |
|||
<th width="40">Hits</th> |
|||
<th>Query</th> |
|||
</tr> |
|||
</thead> |
|||
<tbody> |
|||
{% for link in referrers["referrers"] %} |
|||
{% if link %} |
|||
<tr> |
|||
<td align="center" valign="top"> |
|||
{{ link[1] }} |
|||
</td> |
|||
<td align="left" valign="top" style="position: relative"> |
|||
{{ link[0]|safe }} |
|||
</td> |
|||
</tr> |
|||
{% endif %} |
|||
{% endfor %} |
|||
</tbody> |
|||
</table> |
|||
</div> |
|||
|
|||
<h2 id="recent">25 Most Recent Links</h2> |
|||
|
|||
<ol> |
|||
{% for item in referrers["recent"] %} |
|||
<li>{{ item }}</li> |
|||
{% endfor %} |
|||
</ol> |
|||
|
|||
{% endblock %} |
|||
@ -0,0 +1,54 @@ |
|||
{% extends "layout.html" %} |
|||
{% block title %}Visitor History{% endblock %} |
|||
|
|||
{% block content %} |
|||
|
|||
<h1>Visitor History</h1> |
|||
|
|||
Unique visitors and hit counts have been logged on this site since |
|||
{{ history["oldest"] }}.<p> |
|||
|
|||
The most unique visitors on this site in one day has been |
|||
{{ history["most_unique"][1] }} on {{ history["most_unique"][0] }}. The most |
|||
hits total in one day has been {{ history["most_hits"][1] }} on |
|||
{{ history["most_hits"][0] }}.<p> |
|||
|
|||
Here is a full list of hits over time. Percentages are relative to the current |
|||
records.<p> |
|||
|
|||
<table class="table" width="100%" border="0" cellspacing="2" cellpadding="2"> |
|||
<thead> |
|||
<tr> |
|||
<th width="20">Date</th> |
|||
<th>Graph</th> |
|||
<th width="250">Details</th> |
|||
</tr> |
|||
</thead> |
|||
<tbody> |
|||
{% for date in history["traffic"]|reverse %} |
|||
<tr> |
|||
<td align="center" valign="middle" rowspan="2"> |
|||
{{ date["date"] }} |
|||
</td> |
|||
<td align="left" valign="middle"> |
|||
{% set pct = (date["unique"] / history["most_unique"][1]) * 100 %} |
|||
<div class="visitor-graph unique" style="width: {{ pct|int }}%"></div> |
|||
</td> |
|||
<td align="left" valign="middle"> |
|||
Unique: {{ date["unique"] }} ({{ pct|int }}%) |
|||
</td> |
|||
</tr> |
|||
<tr> |
|||
<td align="left" valign="middle"> |
|||
{% set pct = (date["unique"] / history["most_unique"][1]) * 100 %} |
|||
<div class="visitor-graph hits" style="width: {{ pct|int }}%"></div> |
|||
</td> |
|||
<td align="left" valign="middle"> |
|||
Hits: {{ date["hits"] }} ({{ pct|int }}%) |
|||
</td> |
|||
</tr> |
|||
{% endfor %} |
|||
</tbody> |
|||
</table> |
|||
|
|||
{% endblock %} |
|||
|
After Width: | Height: | Size: 306 B |
|
After Width: | Height: | Size: 306 B |
Loading…
Reference in new issue