MediaWiki:Louupdater.js

/* List of Untergangers Updater * v0.1 * by mfaizsyayhmi, 2017 * another attempt at doing a data scraper, now that data have been separated from markup * still WIP, expect a lot of runtime errors! */

// namespace window.lou = window.lou || {};

lou.cfg = { rs: '§', fs: '‖', dataUrl: '/wiki/User:Mfaizsyahmi/testdata?action=render', map: ['name','reg','ctry','vids','views','subs','sbgrade','status','yt','notes'], widths: [ 25,   3,     2,     4,     10,     7,        2,       1], defProxyUrl: 'http://allorigins.us', key: "AIzaSyD8-62lEUulbGAFC7NtWNHW1kVvh6JLqq4" // for use on Hitler Parody Wiki only! };

// working vars lou.dataText = null; lou.data = null; lou.ajaxQueue = []; // holds all the ajax settings objects

// runs on page load lou.init = function { // abort if not on the correct page if (mw.config.get('wgPageName')!=='Special:LoUUpdater') return; // update title $('h1, title').text('List of Untergangers Updater'); // build interface var content = $('#WikiaArticle'); content.html(''); $(' ').text('Step 1: Load data').appendTo(content); $(' ').text('Load data').on('click', function{		$('#s1stat').text('Loading data... (if it takes too long it might have failed)');		$.get(lou.cfg.dataUrl, function(d) { lou.dataText = d;			lou.data = lou.parseData(lou.dataText); $('#count').val(lou.data.length); $('#s1stat').text(lou.data.length + ' records loaded.'); })	}).appendTo(content); $(' ').attr('id','s1stat').appendTo(content); $(' ').text('Step 2: Update entries').appendTo(content); var p = $(' ').append('Offset by '); $(' ').prop({type:'input', id: 'offset', value: 0}).css('width','3em').appendTo(p); p.append(' and update '); $(' ').prop({type:'input', id: 'count', value: 0}).css('width','3em').appendTo(p); p.append(' entries. Use the following CORS proxy server (an AnyOrigin clone): ') $(' ').prop({type:'input', id: 'proxyUrl', value: lou.cfg.defProxyUrl}).appendTo(p); p.appendTo(content); $(' ').prop('id', 'startBtn').text('Start').on('click', function{		lou.startUpdate;		$(this).prop('disabled',true);	}).appendTo(content); $(' ').prop('id', 'pauseBtn').text('Pause').on('click', function{		$(this).addClass('pausing');	}).appendTo(content); $(' ').prop('id', 'resumeBtn').text('Resume').on('click', function{		lou.next;		$(this).hide;		$('#pauseBtn').show;	}).hide.appendTo(content); $(' ').prop('id', 'cancelBtn').text('Cancel').on('click', function{		$(this).addClass('cancelling');	}).appendTo(content); $(' ').attr('id','s2stat').text('Not started').appendTo(content); $(' ').prop({id: 'progress', value: 0}) .css({width: '100%'}) .appendTo(content); $(' ').attr('id','s2log').appendTo(content); $(' ').text('Output').appendTo(content); $(' ').text('The full list with updated parts below. Please check for any request errors (listed below the progress bar), and that the formatting is correct.').appendTo(content); $(' ').attr('id', 'output').css({'max-height': '30em', 'max-width':'100%'}).appendTo(content); } lou.init;

lou.parseData = function(dataText) { var rawArray = dataText.split(lou.cfg.rs); rawArray.pop; rawArray.shift; var outArray = []; for (var i=0; i1) { // field is named recordObj[kv[0]] = kv[kv.length-1]; } else { recordObj[lou.cfg.map[n]] = kv[0]; n++; }		}		outArray.push(recordObj); }	return outArray; } lou.formatData = function(dataArray) { var strArr = []; for (var i=0; i-1) { // property mapped to index recordArr[lou.cfg.map.indexOf(prop)] = lou.pad(dataArray[i][prop], prop); } else if (dataArray[i].hasOwnProperty(prop)) { // prop not mapped to index recordArr.push(prop+'='+dataArray[i][prop]); }		}		strArr.push(recordArr.join(lou.cfg.fs)); }	return lou.cfg.rs + strArr.join('\n' + lou.cfg.rs); }

lou.pad = function(str, prop) { if(typeof str==='undefined') return; var width = lou.cfg.widths[lou.cfg.map.indexOf(prop)]; //if (prop==='name') {console.log(str.length, width)} // debug if (prop==='name' && str.length>width) { return str + '\n' + ' '.repeat(width+1); } else if ( isNaN(str.replace(/,/g,'')) ) { return str.padEnd(width); } else { return str.padStart(width); } }

lou.startUpdate = function { lou.ajaxQueue = []; // clear the queue var offset = $('#offset').val, count = $('#count').val, proxyUrl = $('#proxyUrl').val, limit = offset + count, idList = []; // reset some ui stuff $('#s2log').text(''); lou.s2log('Preparing requests'); document.getElementById('progress').max = 0; document.getElementById('progress').value = 0; // go through the records and push ajax objects for (var i=offset; i<lou.data.length && i<limit; i++) { var recordObj = lou.data[i], skip = false; u; if (recordObj.hasOwnProperty('noajax') || !recordObj.hasOwnProperty('yt')) { // no yt or noajax set; skip skip = true; } else if (recordObj.hasOwnProperty('yt') && recordObj.yt==='~') { recordObj._yt = recordObj.name.match(/\[\[([^|\]]+).*\]\]/)[1]; u = recordObj._yt; lou.pushYtAjax(false, u); } else if (lou.isChanId(recordObj.yt)) { u = recordObj.yt; idList.push(u); } else { // duplicated since the original is going to be replaced with channel id and SB still uses username recordObj._yt = recordObj.yt; u = recordObj.yt; lou.pushYtAjax(false, u); }		if (!skip) lou.pushSbAjax(proxyUrl, u); if (idList.length===50) { lou.pushYtAjax(true, idList); idList = []; }	}	// start the queue processing lou.next(0); }

lou.pushYtAjax = function(isIdList, data) { var dataObj = {part: 'id,statistics', key: lou.cfg.key}; var progressworth = 1; if(isIdList) { dataObj.id = data.join(','); dataObj.maxResults = 50; progressworth = data.length; } else { dataObj.forUsername = data; }	lou.ajaxQueue.push({		type: 'GET',		url: 'https://www.googleapis.com/youtube/v3/channels',		data: dataObj,		dataType: 'json',		success: function(d, s, xhr) {			//console.log(typeof dataObj)//debug			if(dataObj.forUsername) {				var recordObj = lou.data.find( function(obj) { if(obj.yt===dataObj.forUsername || obj._yt===dataObj.forUsername) return obj; });				if (recordObj && d.items.length) {					recordObj.yt = d.items[0].id,					recordObj.vids = d.items[0].statistics.videoCount,					recordObj.views = d.items[0].statistics.viewCount,					recordObj.subs = d.items[0].statistics.subscriberCount;				} else {					lou.s2log('Error - username mismatch for YT: ' +dataObj.forUsername+ '. Channel might have been deleted.', true);				}			} else {				for(var i=0; i<d.items.length; i++) {					var recordObj = lou.data.find( function(obj) { if(obj.yt===d.items[i].id) return obj; });					if(recordObj) {						recordObj.yt = d.items[i].id,						recordObj.vids = d.items[i].statistics.videoCount,						recordObj.views = d.items[i].statistics.viewCount,						recordObj.subs = d.items[i].statistics.subscriberCount;					}				}			}		},		error: function(xhr, type, stat) {			if (isIdList) {				lou.s2log('Error requesting YT for batch user data. Type: ' +type+ ', Status: ' +stat, true);			} else {				lou.s2log('Error requesting YT for user ' +data+ '. Type: ' +type+ ', Status: ' +stat, true);			}		},		complete: function {			lou.next(progressworth);		}	}); document.getElementById('progress').max +=progressworth; } lou.pushSbAjax = function(proxyUrl, u) { console.log(u)//debug lou.ajaxQueue.push({		type: 'GET',		url: proxyUrl + '/get?url=' + encodeURIComponent('http://widget.socialblade.com/widget?v=2&u='+u),		dataType: 'json',		success: function(d) {			var recordObj = lou.data.find( function(obj) { if(obj.yt===u || obj._yt===u) return obj; })			if (recordObj) {				recordObj.sbgrade = $('.widgetGrade span', d.contents).text;				console.log(recordObj.sbgrade);//debug			} else {				lou.s2log('Error - username mismatch for SB: ' + dataObj.forUsername);			}		}, 		error: function(xhr, type, stat) {			lou.s2log('Error requesting SB for user ' +u+ '. Type: ' +type+ ', Status: '+stat, true);		},		complete: function {			lou.next(1);		}	}); document.getElementById('progress').max +=1; }

lou.next = function(progress) { if (!progress) progress = 1; document.getElementById('progress').value +=progress; if ($('#pauseBtn').is('.pausing')) { $('#pauseBtn').removeClass('pausing').text('Pause').hide; $('#resumeBtn').show; lou.s2log('Paused.'); } else if ($('#cancelBtn').is('.cancelling')) { $('#cancelBtn').removeClass('cancelling').text('Cancel'); $('#startBtn').prop('disabled', false); lou.s2log('Cancelled.'); lou.formatData(lou.data); } else if (lou.ajaxQueue.length===0) { lou.s2log('Completed.'); $('#startBtn').prop('disabled', false); $('#output').text(lou.formatData(lou.data)); } else { lou.s2log('Processing request (' +lou.ajaxQueue.length+ ' remaining)'); setTimeout( function {			$.ajax(lou.ajaxQueue.shift);		}, 20); } }

lou.s2log = function(str, log) { $('#s2stat').text(str); if (log) $('#s2log').append(str + ' '); }

lou.isChanId = function(str) { return (str!== undefined && str.length===24 && str.substr(0,2)==='UC'); }