archive
/
nuTensor
mirror of https://github.com/geekprojects/nuTensor.git
1
0
Fork 0
Code Issues Releases Wiki Activity

code review: simplify code for counting distinct URLs

Browse Source
pull/2/head
gorhill 7 years ago
parent 67e7c8a5bf
commit ebac131247
No known key found for this signature in database
GPG Key ID: 25E1490B761470C2
6 changed files with 159 additions and 403 deletions
Show Stats Download Patch File Download Diff File

46
src/js/messaging.js
Unescape Escape View File

@ -211,16 +211,12 @@ var matrixSnapshot = function(pageStore, details) {
var desHostname;
var row, typeIndex;
var anyIndex = headerIndices.get('*');
var pos, count;
var pageRequests = pageStore.requests;
var reqKeys = pageRequests.getRequestKeys();
var iReqKey = reqKeys.length;
var pos;
while ( iReqKey-- ) {
reqKey = reqKeys[iReqKey];
reqType = pageRequests.typeFromRequestKey(reqKey);
reqHostname = pageRequests.hostnameFromRequestKey(reqKey);
for ( var entry of pageStore.requests.hostnameTypeCells ) {
pos = entry[0].indexOf(' ');
reqHostname = entry[0].slice(0, pos);
reqType = entry[0].slice(pos + 1);
// rhill 2013-10-23: hostname can be empty if the request is a data url
// https://github.com/gorhill/httpswitchboard/issues/26
if ( reqHostname === '' ) {
@ -230,39 +226,31 @@ var matrixSnapshot = function(pageStore, details) {
// We want rows of self and ancestors
desHostname = reqHostname;
for ( ;; ) {
for (;;) {
// If row exists, ancestors exist
if ( r.rows.hasOwnProperty(desHostname) !== false ) {
break;
}
if ( r.rows.hasOwnProperty(desHostname) !== false ) { break; }
r.rows[desHostname] = new RowSnapshot(r.scope, desHostname, reqDomain);
r.rowCount += 1;
if ( desHostname === reqDomain ) {
break;
}
if ( desHostname === reqDomain ) { break; }
pos = desHostname.indexOf('.');
if ( pos === -1 ) {
break;
}
if ( pos === -1 ) { break; }
desHostname = desHostname.slice(pos + 1);
}
count = entry[1].size;
typeIndex = headerIndices.get(reqType);
row = r.rows[reqHostname];
row.counts[typeIndex] += 1;
row.counts[anyIndex] += 1;
row.counts[typeIndex] += count;
row.counts[anyIndex] += count;
row = r.rows[reqDomain];
row.totals[typeIndex] += 1;
row.totals[anyIndex] += 1;
row.totals[typeIndex] += count;
row.totals[anyIndex] += count;
row = r.rows['*'];
row.totals[typeIndex] += 1;
row.totals[anyIndex] += 1;
row.totals[typeIndex] += count;
row.totals[anyIndex] += count;
}
r.diff = µm.tMatrix.diff(µm.pMatrix, r.hostname, Object.keys(r.rows));
r.diff = µm.tMatrix.diff(µm.pMatrix, r.hostname, r.rowCount + 1);
return r;
};

483
src/js/pagestats.js
Unescape Escape View File

@ -25,393 +25,162 @@
/*******************************************************************************
A PageRequestStore object is used to store net requests in two ways:
A PageRequestStats object is used to store distinct network requests.
This is used to:
To record distinct net requests
- remember which hostname/type were seen
- count the number of distinct URLs for any given hostname-type pair
**/
µMatrix.PageRequestStats = (function() {
µMatrix.pageRequestStatsFactory = (function() {
/******************************************************************************/
// Caching useful global vars
var µm = µMatrix;
var µmuri = null;
/******************************************************************************/
// Hidden vars
var typeToCode = {
'doc' : 'a',
'frame' : 'b',
'css' : 'c',
'script': 'd',
'image' : 'e',
'media' : 'f',
'xhr' : 'g',
'other' : 'h',
'cookie': 'i'
};
var codeToType = {
'a': 'doc',
'b': 'frame',
'c': 'css',
'd': 'script',
'e': 'image',
'f': 'media',
'g': 'xhr',
'h': 'other',
'i': 'cookie'
};
var µm = µMatrix;
var µmuri;
var pageRequestStoreJunkyard = [];
/******************************************************************************/
// It's just a dict-based "packer"
var stringPacker = {
codeGenerator: 1,
codeJunkyard: [],
mapStringToEntry: {},
mapCodeToString: {},
Entry: function(code) {
this.count = 0;
this.code = code;
},
remember: function(code) {
if ( code === '' ) {
return;
}
var s = this.mapCodeToString[code];
if ( s ) {
var entry = this.mapStringToEntry[s];
entry.count++;
}
},
// Ref: Given a URL, returns a (somewhat) unique 32-bit value
// Based on: FNV32a
// http://www.isthe.com/chongo/tech/comp/fnv/index.html#FNV-reference-source
// The rest is custom, suited for µMatrix.
forget: function(code) {
if ( code === '' ) {
return;
var uidFromURL = function(uri) {
var hint = 0x811c9dc5;
var i = uri.length;
while ( i-- ) {
hint ^= uri.charCodeAt(i) | 0;
hint += (hint<<1) + (hint<<4) + (hint<<7) + (hint<<8) + (hint<<24) | 0;
hint >>>= 0;
}
var s = this.mapCodeToString[code];
if ( s ) {
var entry = this.mapStringToEntry[s];
entry.count--;
if ( !entry.count ) {
// console.debug('stringPacker > releasing code "%s" (aka "%s")', code, s);
this.codeJunkyard.push(entry);
delete this.mapCodeToString[code];
delete this.mapStringToEntry[s];
return hint;
};
var PageRequestStats = function() {
this.hostnameTypeCells = new Map();
};
PageRequestStats.prototype = {
dispose: function() {
this.hostnameTypeCells.clear();
if ( pageRequestStoreJunkyard.length < 8 ) {
pageRequestStoreJunkyard.push(this);
}
}
},
pack: function(s) {
var entry = this.entryFromString(s);
if ( !entry ) {
return '';
}
return entry.code;
},
unpack: function(packed) {
return this.mapCodeToString[packed] || '';
},
stringify: function(code) {
if ( code <= 0xFFFF ) {
return String.fromCharCode(code);
}
return String.fromCharCode(code >>> 16) + String.fromCharCode(code & 0xFFFF);
},
entryFromString: function(s) {
if ( s === '' ) {
return null;
}
var entry = this.mapStringToEntry[s];
if ( !entry ) {
entry = this.codeJunkyard.pop();
if ( !entry ) {
entry = new this.Entry(this.stringify(this.codeGenerator++));
},
createEntryIfNotExists: function(url, type) {
var hn = µmuri.hostnameFromURI(url),
key = hn + ' ' + type,
uids = this.hostnameTypeCells.get(key);
if ( uids === undefined ) {
this.hostnameTypeCells.set(key, (uids = new Set()));
} else {
// console.debug('stringPacker > recycling code "%s" (aka "%s")', entry.code, s);
entry.count = 0;
if ( uids.size > 99 ) { return false; }
}
this.mapStringToEntry[s] = entry;
this.mapCodeToString[entry.code] = s;
var uid = uidFromURL(url);
if ( uids.has(uid) ) { return false; }
uids.add(uid);
return true;
}
return entry;
}
};
/******************************************************************************/
var PageRequestStats = function() {
this.requests = {};
if ( !µmuri ) {
µmuri = µm.URI;
}
};
/******************************************************************************/
PageRequestStats.prototype.init = function() {
return this;
};
/******************************************************************************/
var pageRequestStoreJunkyard = [];
var pageRequestStoreFactory = function() {
var pageRequestStore = pageRequestStoreJunkyard.pop();
if ( pageRequestStore ) {
pageRequestStore.init();
} else {
pageRequestStore = new PageRequestStats();
}
return pageRequestStore;
};
/******************************************************************************/
PageRequestStats.prototype.disposeOne = function(reqKey) {
if ( this.requests[reqKey] ) {
delete this.requests[reqKey];
forgetRequestKey(reqKey);
}
};
/******************************************************************************/
};
PageRequestStats.prototype.dispose = function() {
var requests = this.requests;
for ( var reqKey in requests ) {
if ( requests.hasOwnProperty(reqKey) === false ) {
continue;
return function pageRequestStatsFactory() {
if ( pageRequestStoreJunkyard.length !== 0 ) {
return pageRequestStoreJunkyard.pop();
}
stringPacker.forget(reqKey.slice(3));
}
this.requests = {};
if ( pageRequestStoreJunkyard.length < 8 ) {
pageRequestStoreJunkyard.push(this);
}
};
/******************************************************************************/
// Request key:
// index: 0123
// THHN
// ^^ ^
// || |
// || +--- short string code for hostname (dict-based)
// |+--- FNV32a hash of whole URI (irreversible)
// +--- single char code for type of request
var makeRequestKey = function(uri, reqType) {
// Ref: Given a URL, returns a unique 4-character long hash string
// Based on: FNV32a
// http://www.isthe.com/chongo/tech/comp/fnv/index.html#FNV-reference-source
// The rest is custom, suited for µMatrix.
var hint = 0x811c9dc5;
var i = uri.length;
while ( i-- ) {
hint ^= uri.charCodeAt(i) | 0;
hint += (hint<<1) + (hint<<4) + (hint<<7) + (hint<<8) + (hint<<24) | 0;
hint >>>= 0;
}
var key = typeToCode[reqType] || 'z';
return key +
String.fromCharCode(hint >>> 22, hint >>> 12 & 0x3FF, hint & 0xFFF) +
stringPacker.pack(µmuri.hostnameFromURI(uri));
};
/******************************************************************************/
var rememberRequestKey = function(reqKey) {
stringPacker.remember(reqKey.slice(4));
};
var forgetRequestKey = function(reqKey) {
stringPacker.forget(reqKey.slice(4));
};
/******************************************************************************/
// Exported
var hostnameFromRequestKey = function(reqKey) {
return stringPacker.unpack(reqKey.slice(4));
};
PageRequestStats.prototype.hostnameFromRequestKey = hostnameFromRequestKey;
var typeFromRequestKey = function(reqKey) {
return codeToType[reqKey.charAt(0)];
};
PageRequestStats.prototype.typeFromRequestKey = typeFromRequestKey;
/******************************************************************************/
PageRequestStats.prototype.createEntryIfNotExists = function(url, type) {
var reqKey = makeRequestKey(url, type);
if ( this.requests[reqKey] ) {
return false;
}
rememberRequestKey(reqKey);
this.requests[reqKey] = Date.now();
return true;
};
/******************************************************************************/
PageRequestStats.prototype.getRequestKeys = function() {
return Object.keys(this.requests);
};
/******************************************************************************/
PageRequestStats.prototype.getRequestDict = function() {
return this.requests;
};
/******************************************************************************/
// Export
return {
factory: pageRequestStoreFactory,
hostnameFromRequestKey: hostnameFromRequestKey,
typeFromRequestKey: typeFromRequestKey
};
/******************************************************************************/
if ( µmuri === undefined ) { µmuri = µm.URI; }
return new PageRequestStats();
};
})();
/******************************************************************************/
/******************************************************************************/
µMatrix.PageStore = (function() {
µMatrix.pageStoreFactory = (function() {
/******************************************************************************/
var µm = µMatrix;
var pageStoreJunkyard = [];
/******************************************************************************/
var µm = µMatrix;
var pageStoreJunkyard = [];
var pageStoreFactory = function(tabContext) {
var entry = pageStoreJunkyard.pop();
if ( entry ) {
return entry.init(tabContext);
function PageStore(tabContext) {
this.requestStats = µm.requestStatsFactory();
this.off = false;
this.init(tabContext);
}
return new PageStore(tabContext);
};
/******************************************************************************/
function PageStore(tabContext) {
this.requestStats = µm.requestStatsFactory();
this.off = false;
this.init(tabContext);
}
/******************************************************************************/
PageStore.prototype.init = function(tabContext) {
this.tabId = tabContext.tabId;
this.rawUrl = tabContext.rawURL;
this.pageUrl = tabContext.normalURL;
this.pageHostname = tabContext.rootHostname;
this.pageDomain = tabContext.rootDomain;
this.title = '';
this.requests = µm.PageRequestStats.factory();
this.domains = {};
this.allHostnamesString = ' ';
this.requestStats.reset();
this.distinctRequestCount = 0;
this.perLoadAllowedRequestCount = 0;
this.perLoadBlockedRequestCount = 0;
this.incinerationTimer = null;
this.mtxContentModifiedTime = 0;
this.mtxCountModifiedTime = 0;
return this;
};
/******************************************************************************/
PageStore.prototype.dispose = function() {
this.requests.dispose();
this.rawUrl = '';
this.pageUrl = '';
this.pageHostname = '';
this.pageDomain = '';
this.title = '';
this.domains = {};
this.allHostnamesString = ' ';
if ( this.incinerationTimer !== null ) {
clearTimeout(this.incinerationTimer);
this.incinerationTimer = null;
}
if ( pageStoreJunkyard.length < 8 ) {
pageStoreJunkyard.push(this);
}
};
/******************************************************************************/
PageStore.prototype.recordRequest = function(type, url, block) {
if ( !this.requests.createEntryIfNotExists(url, type, block) ) {
return;
}
// Count blocked/allowed requests
this.requestStats.record(type, block);
// https://github.com/gorhill/httpswitchboard/issues/306
// If it is recorded locally, record globally
µm.requestStats.record(type, block);
µm.updateBadgeAsync(this.tabId);
if ( block !== false ) {
this.perLoadBlockedRequestCount++;
} else {
this.perLoadAllowedRequestCount++;
}
var hostname = µm.URI.hostnameFromURI(url);
PageStore.prototype = {
init: function(tabContext) {
this.tabId = tabContext.tabId;
this.rawUrl = tabContext.rawURL;
this.pageUrl = tabContext.normalURL;
this.pageHostname = tabContext.rootHostname;
this.pageDomain = tabContext.rootDomain;
this.title = '';
this.requests = µm.pageRequestStatsFactory();
this.domains = {};
this.allHostnamesString = ' ';
this.requestStats.reset();
this.distinctRequestCount = 0;
this.perLoadAllowedRequestCount = 0;
this.perLoadBlockedRequestCount = 0;
this.incinerationTimer = null;
this.mtxContentModifiedTime = 0;
this.mtxCountModifiedTime = 0;
return this;
},
dispose: function() {
this.requests.dispose();
this.rawUrl = '';
this.pageUrl = '';
this.pageHostname = '';
this.pageDomain = '';
this.title = '';
this.domains = {};
this.allHostnamesString = ' ';
if ( this.incinerationTimer !== null ) {
clearTimeout(this.incinerationTimer);
this.incinerationTimer = null;
}
if ( pageStoreJunkyard.length < 8 ) {
pageStoreJunkyard.push(this);
}
},
recordRequest: function(type, url, block) {
if ( this.requests.createEntryIfNotExists(url, type) === false ) {
return;
}
this.distinctRequestCount++;
this.mtxCountModifiedTime = Date.now();
// Count blocked/allowed requests
this.requestStats.record(type, block);
if ( this.domains.hasOwnProperty(hostname) === false ) {
this.domains[hostname] = true;
this.allHostnamesString += hostname + ' ';
this.mtxContentModifiedTime = Date.now();
}
// https://github.com/gorhill/httpswitchboard/issues/306
// If it is recorded locally, record globally
µm.requestStats.record(type, block);
µm.updateBadgeAsync(this.tabId);
// console.debug("pagestats.js > PageStore.recordRequest(): %o: %s @ %s", this, type, url);
};
if ( block !== false ) {
this.perLoadBlockedRequestCount++;
} else {
this.perLoadAllowedRequestCount++;
}
/******************************************************************************/
var hostname = µm.URI.hostnameFromURI(url);
return {
factory: pageStoreFactory
};
this.distinctRequestCount++;
this.mtxCountModifiedTime = Date.now();
/******************************************************************************/
if ( this.domains.hasOwnProperty(hostname) === false ) {
this.domains[hostname] = true;
this.allHostnamesString += hostname + ' ';
this.mtxContentModifiedTime = Date.now();
}
}
};
return function pageStoreFactory(tabContext) {
var entry = pageStoreJunkyard.pop();
if ( entry ) {
return entry.init(tabContext);
}
return new PageStore(tabContext);
};
})();
/******************************************************************************/

28
src/js/popup.js
Unescape Escape View File

@ -437,10 +437,16 @@ function updateMatrixCounts() {
count = rows[expandos.hostname][counts][headerIndices.get(expandos.reqType)];
if ( count === expandos.count ) { continue; }
expandos.count = count;
matCell.textContent = count ? count : '\u00A0';
matCell.textContent = cellTextFromCount(count);
}
}
function cellTextFromCount(count) {
if ( count === 0 ) { return '\u00A0'; }
if ( count < 100 ) { return count; }
return '99+';
}
/******************************************************************************/
// Update color of matrix cells(s)
@ -686,16 +692,13 @@ function renderMatrixMetaCellDomain(cell, domain) {
}
function renderMatrixCellType(cell, hostname, type, count) {
var expandos = expandosFromNode(cell);
var node = cell.nodeAt(0),
expandos = expandosFromNode(node);
expandos.hostname = hostname;
expandos.reqType = type;
expandos.count = count;
addCellClass(cell.nodeAt(0), hostname, type);
if ( count ) {
cell.text(count);
} else {
cell.text('\u00A0');
}
addCellClass(node, hostname, type);
node.textContent = cellTextFromCount(count);
}
function renderMatrixCellTypes(cells, hostname, countName) {
@ -742,16 +745,13 @@ function makeMatrixMetaRowDomain(domain) {
function renderMatrixMetaCellType(cell, count) {
// https://github.com/gorhill/uMatrix/issues/24
// Don't forget to reset cell properties
var expandos = expandosFromNode(cell);
var node = cell.nodeAt(0),
expandos = expandosFromNode(node);
expandos.hostname = '';
expandos.reqType = '';
expandos.count = count;
cell.addClass('t1');
if ( count ) {
cell.text(count);
} else {
cell.text('\u00A0');
}
node.textContent = cellTextFromCount(count);
}
function makeMatrixMetaRow(totals) {

2
src/js/start.js
Unescape Escape View File

@ -187,7 +187,7 @@ var onPSLReady = function() {
// rhill 2013-11-24: bind behind-the-scene virtual tab/url manually, since the
// normal way forbid binding behind the scene tab.
// https://github.com/gorhill/httpswitchboard/issues/67
µm.pageStores[vAPI.noTabId] = µm.PageStore.factory(µm.tabContextManager.mustLookup(vAPI.noTabId));
µm.pageStores[vAPI.noTabId] = µm.pageStoreFactory(µm.tabContextManager.mustLookup(vAPI.noTabId));
µm.pageStores[vAPI.noTabId].title = vAPI.i18n('statsPageDetailedBehindTheScenePage');
vAPI.tabs.getAll(onTabsReady);

2
src/js/tab.js
Unescape Escape View File

@ -445,7 +445,7 @@ vAPI.tabs.registerListeners();
// Try to resurrect first.
pageStore = this.resurrectPageStore(tabId, normalURL);
if ( pageStore === null ) {
pageStore = this.PageStore.factory(tabContext);
pageStore = this.pageStoreFactory(tabContext);
}
this.pageStores[tabId] = pageStore;
this.updateTitle(tabId);

1
src/js/traffic.js
Unescape Escape View File

@ -374,7 +374,6 @@ var requestTypeNormalizer = {
'media' : 'media',
'object' : 'media',
'other' : 'other',
'ping' : 'ping',
'script' : 'script',
'stylesheet' : 'css',
'sub_frame' : 'frame',

Write Preview
Loading…
Cancel
Save