parent
a1b2550c0b
commit
67a30e2471
16 changed files with 377 additions and 31 deletions
@ -0,0 +1,175 @@ |
||||
# -*- coding: utf-8 -*- |
||||
|
||||
"""Visitor tracking models.""" |
||||
|
||||
import time |
||||
import requests |
||||
|
||||
import rophako.jsondb as JsonDB |
||||
from rophako.utils import remote_addr, pretty_time, server_name |
||||
|
||||
def track_visit(request, session): |
||||
"""Main logic to track and log visitor details.""" |
||||
|
||||
# Get their tracking cookie value. The value will either be their HTTP |
||||
# referrer (if exists and valid) or else a "1". |
||||
cookie = session.get("tracking") |
||||
addr = remote_addr() |
||||
values = dict() # Returnable traffic values |
||||
|
||||
# Log hit counts. We need four kinds: |
||||
# - Unique today - Unique total |
||||
# - Hits today - Hits total |
||||
today = pretty_time("%Y-%m-%d", time.time()) |
||||
files = { |
||||
"unique/{}".format(today) : "unique_today", |
||||
"unique/total" : "unique_total", |
||||
"hits/{}".format(today) : "hits_today", |
||||
"hits/total" : "hits_total", |
||||
} |
||||
|
||||
# Go through the hit count files. Update them only if their tracking |
||||
# cookie was not present. |
||||
for file, key in files.items(): |
||||
dbfile = "traffic/{}".format(file) |
||||
if file.startswith("hits"): |
||||
# Hit file is just a simple counter. |
||||
db = dict(hits=0) |
||||
if JsonDB.exists(dbfile): |
||||
db = JsonDB.get(dbfile) |
||||
|
||||
# Update it? |
||||
if not cookie: |
||||
db["hits"] += 1 |
||||
JsonDB.commit(dbfile, db) |
||||
|
||||
# Store the copy. |
||||
values[key] = db["hits"] |
||||
else: |
||||
# Unique file is a collection of IP addresses. |
||||
db = dict() |
||||
if JsonDB.exists(dbfile): |
||||
db = JsonDB.get(dbfile) |
||||
|
||||
# Update with their IP? |
||||
if not cookie and not addr in db: |
||||
db[addr] = time.time() |
||||
JsonDB.commit(dbfile, db) |
||||
|
||||
# Store the copy. |
||||
values[key] = len(db.keys()) |
||||
|
||||
# Log their HTTP referrer. |
||||
referrer = "1" |
||||
if request.referrer: |
||||
# Branch and check this. |
||||
referrer = log_referrer(request, request.referrer) |
||||
if not referrer: |
||||
# Wasn't a valid referrer. |
||||
referrer = "1" |
||||
|
||||
# Set their tracking cookie. |
||||
if not cookie: |
||||
cookie = referrer |
||||
session["tracking"] = cookie |
||||
|
||||
return values |
||||
|
||||
|
||||
def log_referrer(request, link): |
||||
"""Double check the referring URL.""" |
||||
|
||||
# Ignore if same domain. |
||||
if link.startswith(request.url_root): |
||||
print "Referrer is same host!" |
||||
return None |
||||
|
||||
# See if the URL really links back to us. |
||||
hostname = server_name() |
||||
r = requests.get(link) |
||||
if hostname in r.text: |
||||
# Log it. |
||||
db = list() |
||||
if JsonDB.exists("traffic/referrers"): |
||||
# Don't cache the result -- the list can get huge! |
||||
db = JsonDB.get("traffic/referrers", cache=False) |
||||
db.append(link) |
||||
JsonDB.commit("traffic/referrers", db, cache=False) |
||||
return link |
||||
|
||||
return None |
||||
|
||||
|
||||
def get_visitor_details(): |
||||
"""Retrieve detailed visitor information for the frontend.""" |
||||
result = { |
||||
"traffic": [], # Historical traffic data |
||||
"most_unique": [ "0000-00-00", 0 ], # Day with the most unique |
||||
"most_hits": [ "0000-00-00", 0 ], # Day with the most hits |
||||
"oldest": None, # Oldest day on record. |
||||
} |
||||
|
||||
# List all the documents. |
||||
hits = JsonDB.list_docs("traffic/hits") |
||||
for date in sorted(hits): |
||||
if date == "total": continue |
||||
if not result["oldest"]: |
||||
result["oldest"] = date |
||||
|
||||
# Get the DBs. |
||||
hits_db = JsonDB.get("traffic/hits/{}".format(date), cache=False) |
||||
uniq_db = JsonDB.get("traffic/unique/{}".format(date), cache=False) |
||||
|
||||
# Most we've seen? |
||||
if hits_db["hits"] > result["most_hits"][1]: |
||||
result["most_hits"] = [ date, hits_db["hits"] ] |
||||
if len(uniq_db.keys()) > result["most_unique"][1]: |
||||
result["most_unique"] = [ date, len(uniq_db.keys()) ] |
||||
|
||||
result["traffic"].append(dict( |
||||
date=date, |
||||
hits=hits_db["hits"], |
||||
unique=len(uniq_db.keys()), |
||||
)) |
||||
|
||||
return result |
||||
|
||||
|
||||
def get_referrers(recent=25): |
||||
"""Retrieve the referrer details. Returns results in this format: |
||||
|
||||
``` |
||||
{ |
||||
referrers: [ |
||||
["http://...", 20], # Pre-sorted by number of hits |
||||
], |
||||
recent: [ recent list ] |
||||
} |
||||
``` |
||||
""" |
||||
db = [] |
||||
if JsonDB.exists("traffic/referrers"): |
||||
db = JsonDB.get("traffic/referrers", cache=False) |
||||
|
||||
# Count the links. |
||||
unique = dict() |
||||
for link in db: |
||||
if not link in unique: |
||||
unique[link] = 1 |
||||
else: |
||||
unique[link] += 1 |
||||
|
||||
# Sort them by popularity. |
||||
result = dict( |
||||
referrers=[], |
||||
recent=[], |
||||
) |
||||
|
||||
sorted_links = sorted(unique.keys(), key=lambda x: unique[x], reverse=True) |
||||
for link in sorted_links: |
||||
result["referrers"].append([ link, unique[link] ]) |
||||
|
||||
recent = 0 - recent |
||||
result["recent"] = db[recent:] |
||||
|
||||
return result |
||||
@ -0,0 +1,43 @@ |
||||
# -*- coding: utf-8 -*- |
||||
|
||||
"""Endpoints for visitor tracking functions.""" |
||||
|
||||
from flask import Blueprint, g |
||||
import re |
||||
|
||||
import rophako.model.tracking as Tracking |
||||
from rophako.utils import template |
||||
|
||||
mod = Blueprint("tracking", __name__, url_prefix="/tracking") |
||||
|
||||
|
||||
@mod.route("/") |
||||
def index(): |
||||
return template("tracking/index.html") |
||||
|
||||
|
||||
@mod.route("/visitors") |
||||
def visitors(): |
||||
g.info["history"] = Tracking.get_visitor_details() |
||||
return template("tracking/visitors.html") |
||||
|
||||
|
||||
@mod.route("/referrers") |
||||
def referrers(): |
||||
g.info["referrers"] = Tracking.get_referrers() |
||||
|
||||
# Filter some of the links. |
||||
for i, link in enumerate(g.info["referrers"]["referrers"]): |
||||
# Clean up useless Google links. |
||||
if "google" in link[0] and re.search(r'/(?:imgres|url|search|translate\w+)?/', link[0]): |
||||
g.info["referrers"]["referrers"][i] = None |
||||
|
||||
# Make the links word-wrap properly. |
||||
filtered = [ |
||||
[ re.sub(r'(.{20})', r'\1<wbr>', x[0]), x[1] ] |
||||
for x in g.info["referrers"]["referrers"] |
||||
if x is not None |
||||
] |
||||
g.info["referrers"]["referrers"] = filtered |
||||
|
||||
return template("tracking/referrers.html") |
||||
@ -0,0 +1,13 @@ |
||||
{% extends "layout.html" %} |
||||
{% block title %}Visitor Tracking{% endblock %} |
||||
|
||||
{% block content %} |
||||
|
||||
<h1>Visitor Tracking</h1> |
||||
|
||||
<ul> |
||||
<li><a href="{{ url_for('tracking.visitors') }}">Unique Visitors & Visits</a></li> |
||||
<li><a href="{{ url_for('tracking.referrers') }}">HTTP Referrers</a></li> |
||||
</ul> |
||||
|
||||
{% endblock %} |
||||
@ -0,0 +1,44 @@ |
||||
{% extends "layout.html" %} |
||||
{% block title %}Referring URLs{% endblock %} |
||||
|
||||
{% block content %} |
||||
|
||||
<h1>Referring URLs</h1> |
||||
|
||||
This table lists the HTTP referrers to this site, in order of popularity. For |
||||
the most recent 25 links, see <a href="#recent">the end of this page</a>.<p> |
||||
|
||||
<div style="height: 450px; overflow: auto"> |
||||
<table class="table" width="100%"> |
||||
<thead> |
||||
<tr> |
||||
<th width="40">Hits</th> |
||||
<th>Query</th> |
||||
</tr> |
||||
</thead> |
||||
<tbody> |
||||
{% for link in referrers["referrers"] %} |
||||
{% if link %} |
||||
<tr> |
||||
<td align="center" valign="top"> |
||||
{{ link[1] }} |
||||
</td> |
||||
<td align="left" valign="top" style="position: relative"> |
||||
{{ link[0]|safe }} |
||||
</td> |
||||
</tr> |
||||
{% endif %} |
||||
{% endfor %} |
||||
</tbody> |
||||
</table> |
||||
</div> |
||||
|
||||
<h2 id="recent">25 Most Recent Links</h2> |
||||
|
||||
<ol> |
||||
{% for item in referrers["recent"] %} |
||||
<li>{{ item }}</li> |
||||
{% endfor %} |
||||
</ol> |
||||
|
||||
{% endblock %} |
||||
@ -0,0 +1,54 @@ |
||||
{% extends "layout.html" %} |
||||
{% block title %}Visitor History{% endblock %} |
||||
|
||||
{% block content %} |
||||
|
||||
<h1>Visitor History</h1> |
||||
|
||||
Unique visitors and hit counts have been logged on this site since |
||||
{{ history["oldest"] }}.<p> |
||||
|
||||
The most unique visitors on this site in one day has been |
||||
{{ history["most_unique"][1] }} on {{ history["most_unique"][0] }}. The most |
||||
hits total in one day has been {{ history["most_hits"][1] }} on |
||||
{{ history["most_hits"][0] }}.<p> |
||||
|
||||
Here is a full list of hits over time. Percentages are relative to the current |
||||
records.<p> |
||||
|
||||
<table class="table" width="100%" border="0" cellspacing="2" cellpadding="2"> |
||||
<thead> |
||||
<tr> |
||||
<th width="20">Date</th> |
||||
<th>Graph</th> |
||||
<th width="250">Details</th> |
||||
</tr> |
||||
</thead> |
||||
<tbody> |
||||
{% for date in history["traffic"]|reverse %} |
||||
<tr> |
||||
<td align="center" valign="middle" rowspan="2"> |
||||
{{ date["date"] }} |
||||
</td> |
||||
<td align="left" valign="middle"> |
||||
{% set pct = (date["unique"] / history["most_unique"][1]) * 100 %} |
||||
<div class="visitor-graph unique" style="width: {{ pct|int }}%"></div> |
||||
</td> |
||||
<td align="left" valign="middle"> |
||||
Unique: {{ date["unique"] }} ({{ pct|int }}%) |
||||
</td> |
||||
</tr> |
||||
<tr> |
||||
<td align="left" valign="middle"> |
||||
{% set pct = (date["unique"] / history["most_unique"][1]) * 100 %} |
||||
<div class="visitor-graph hits" style="width: {{ pct|int }}%"></div> |
||||
</td> |
||||
<td align="left" valign="middle"> |
||||
Hits: {{ date["hits"] }} ({{ pct|int }}%) |
||||
</td> |
||||
</tr> |
||||
{% endfor %} |
||||
</tbody> |
||||
</table> |
||||
|
||||
{% endblock %} |
||||
|
After Width: | Height: | Size: 306 B |
|
After Width: | Height: | Size: 306 B |
Loading…
Reference in new issue