diff options
Diffstat (limited to 'post_fsd_wiki.phantomjs')
| -rw-r--r-- | post_fsd_wiki.phantomjs | 318 |
1 files changed, 222 insertions, 96 deletions
diff --git a/post_fsd_wiki.phantomjs b/post_fsd_wiki.phantomjs index 29b3424..7c18925 100644 --- a/post_fsd_wiki.phantomjs +++ b/post_fsd_wiki.phantomjs @@ -1,29 +1,32 @@ -var BLACKLIST_DATA_FILE = './blacklist-testdata.json' ; -var WIKI_BASE_URL = 'http://localhost/mediawiki/index.php' ; +var BLACKLIST_DATA_FILE = './blacklist-testdata.json' ; +var WIKI_TEMPLATE_FILE = './wiki-template.json' ; +var WIKI_BASE_URL = 'http://localhost/mediawiki/index.php' ; // var WIKI_BASE_URL = 'https://directory.fsf.org/wiki?title=' ; -// var LOGIN_URL = WIKI_URL + '/index.php?title=Special:Userlogin&action=submitlogin' ; -var LOGIN_URL = WIKI_BASE_URL + '?title=Special:UserLogin' ; -var EDIT_URL = WIKI_BASE_URL + '?title=_TITLE_&action=edit&redlink=1' ; -var USAGE_MSG = "USAGE: phantomjs ./post_fsd_wiki.phantomjs <WIKI_LOGIN> <WIKI_PASS>" ; -var TITLE_URL_REGEX = /_TITLE_/ ; -var WIKI_TEXT_BEGIN = '<!-- PARABOLA BLACKLIST BEGIN (DO NOT EDIT) -->Parabola Blacklist Description:' ; -var WIKI_TEXT_END = '<!-- PARABOLA BLACKLIST END -->' ; -var WIKI_TEXT_REGEX = RegExp(WIKI_TEXT_BEGIN + '.*' + WIKI_TEXT_END) ; -var LOGIN_IMPUT_ID = 'wpName1' ; -var PASS_IMPUT_ID = 'wpPassword1' ; -var SUBMIT_IMPUT_ID = 'wpLoginAttempt' ; -var USERNAME_LI_ID = 'pt-userpage' ; -var WIKITEXT_INPUT_ID = 'wpTextbox1' ; -var EDIT_FORM_ID = 'editform' ; -var PACKAGE_NAME_KEY = 'original_package' ; // BLACKLIST_DATA_FILE per blacklist SYNTAX -var REPLACEMENT_KEY = 'libre_replacement' ; // BLACKLIST_DATA_FILE per blacklist SYNTAX -var REFERENCE_KEY = 'ref' ; // BLACKLIST_DATA_FILE per blacklist SYNTAX -var ENTRY_ID_KEY = 'id' ; // BLACKLIST_DATA_FILE per blacklist SYNTAX -var DESCRIPTION_KEY = 'short_description' ; // BLACKLIST_DATA_FILE per blacklist SYNTAX -var BLACKLIST_TAGS_KEY = 'blacklist_tags' // BLACKLIST_DATA_FILE -var STEP_FUNCTION_KEY = 'step-function' ; -var PAGELOAD_WAIT_KEY = 'wait-for-pageload' ; -var STEP_TIMEOUT = 3000 ; var StepTimeout ; +var LOGIN_URL = WIKI_BASE_URL + '?title=Special:UserLogin' ; +var EDIT_URL = WIKI_BASE_URL + '?title=_TITLE_&action=edit' ; +var USAGE_MSG = "USAGE: phantomjs ./post_fsd_wiki.phantomjs <WIKI_LOGIN> <WIKI_PASS>" ; +var TITLE_URL_REGEX = /_TITLE_/ ; +var PARABOLA_ENTRY_HEADER = '== Parabola Blacklist Description ==\n' ; +var WIKI_TEXT_BEGIN = '<!-- PARABOLA BLACKLIST BEGIN (DO NOT EDIT) -->' + PARABOLA_ENTRY_HEADER ; +var WIKI_TEXT_END = '\n<!-- PARABOLA BLACKLIST END -->\n' ; +var WIKI_TEXT_REGEX = RegExp(WIKI_TEXT_BEGIN.replace(/\((.*)\)/ , '\\($1\\)') + '.*' + WIKI_TEXT_END) ; +var LOGIN_IMPUT_ID = 'wpName1' ; +var PASS_IMPUT_ID = 'wpPassword1' ; +var LOGIN_SUBMIT_IMPUT_ID = 'wpLoginAttempt' ; +var USERNAME_LI_ID = 'pt-userpage' ; +var EDIT_TEXT_INPUT_ID = 'wpTextbox1' ; +var EDIT_SUBMIT_INPUT_ID = 'wpSave' ; +var WIKI_TITLE_H1_ID = 'firstHeading' ; +var WIKI_TEXT_DIV_ID = 'mw-content-text' ; +var PACKAGE_NAME_KEY = 'original_package' ; // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[0] +var REPLACEMENT_KEY = 'libre_replacement' ; // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[1] +var REFERENCE_KEY = 'ref' ; // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[2] +var ENTRY_ID_KEY = 'id' ; // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[3] +var DESCRIPTION_KEY = 'short_description' ; // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[4] +var BLACKLIST_TAGS_KEY = 'blacklist_tags' // BLACKLIST_DATA_FILE JSON +var STEP_FUNCTION_KEY = 'step-function' ; +var PAGELOAD_WAIT_KEY = 'wait-for-pageload' ; +var STEP_TIMEOUT = 3000 ; var StepTimeout ; var System = require('system') ; var Args = System.args ; @@ -38,12 +41,12 @@ var ShouldWait = false ; var ShouldQuit = false ; -if (WIKI_LOGIN == '' || WIKI_PASS == '') { DBG(USAGE_MSG) ; phantom.exit() ; } +if (WIKI_LOGIN == '' || WIKI_PASS == '') { LOG(USAGE_MSG) ; phantom.exit() ; } LOG('') ; /* steps */ -function prepare() +function Prepare() { LOG("loading BLACKLIST_DATA_FILE") ; var blacklist_data = require(BLACKLIST_DATA_FILE) ; @@ -53,24 +56,25 @@ function prepare() blacklist_data.forEach(function(package_data) { - var page_title = package_data[PACKAGE_NAME_KEY] ; - var page_text = package_data[DESCRIPTION_KEY ] ; + var page_title = package_data[PACKAGE_NAME_KEY].trim() ; + var page_text = package_data[DESCRIPTION_KEY ].trim() ; if (!page_title || !IsA(page_title , String) || !page_text || !IsA(page_text , String) ) { ForceQuit("invalid package data: " + JSON.stringify(package_data)) ; return ; } LOG("found package: " + page_title) ; - var wiki_page = {} ; - wiki_page[PACKAGE_NAME_KEY] = page_title.trim().replace(' ' , '_') ; - wiki_page[DESCRIPTION_KEY ] = page_text .trim() ; + + var wiki_page = {} ; + wiki_page[PACKAGE_NAME_KEY] = page_title.toLowerCase().replace(/ /g , '_') ; + wiki_page[DESCRIPTION_KEY ] = page_text ; WikiPages.push(wiki_page) ; }) ; IsReady = true ; } -function loadLoginPage() +function LoadLoginPage() { if (WikiPages.length == 0) { ForceQuit("no WikiPages to consider") ; return ; } @@ -86,101 +90,199 @@ function loadLoginPage() LOG("loaded package data: " + page_title) ; } - OpenUrl(LOGIN_URL) ; + GetUrl(LOGIN_URL) ; } -function submitLogin() +function SubmitLogin() { - QuitOnErr(Page.evaluate(function(login_input_id , pass_input_id , submit_input_id , + EvalQuitOnErr(Page.evaluate(function(login_input_id , pass_input_id , submit_input_id , wiki_login , wiki_pass ) { - var login_input = document.getElementById(login_input_id ) ; - var pass_input = document.getElementById(pass_input_id ) ; - var submit_input = document.getElementById(submit_input_id) ; - var login_form = submit_input.form ; + var login_input = document.getElementById(login_input_id ) ; + var pass_input = document.getElementById(pass_input_id ) ; + var submit_button = document.getElementById(submit_input_id) ; + var login_form = submit_button.form ; - if (!login_input || !login_input || !submit_input || !login_form) - return "invalid login page" ; + if (!login_input || login_input.nodeName != 'INPUT' || + !pass_input || pass_input.nodeName != 'INPUT' || + !submit_button || submit_button.nodeName != 'BUTTON' || + !login_form || login_form.nodeName != 'FORM' ) + return "ERROR: invalid login page" ; login_input.value = wiki_login ; pass_input.value = wiki_pass ; login_form.submit() ; - } , LOGIN_IMPUT_ID , PASS_IMPUT_ID , SUBMIT_IMPUT_ID , WIKI_LOGIN , WIKI_PASS)) ; + } , LOGIN_IMPUT_ID , PASS_IMPUT_ID , LOGIN_SUBMIT_IMPUT_ID , WIKI_LOGIN , WIKI_PASS)) ; } -function verifyLogin() +function VerifyLogin() { - QuitOnErr(Page.evaluate(function(username_li_id , wiki_login) + EvalQuitOnErr(Page.evaluate(function(username_li_id , wiki_login) { var username_li = document.getElementById(username_li_id) ; + var username = !!username_li && username_li.textContent.toLowerCase() ; - if (!username_li || username_li.textContent.lowercase != wiki_login.lowercase) - return "login failed" ; + if (!username_li || username_li.nodeName != 'LI' || + username != wiki_login.toLowerCase() ) + return "ERROR: login failed" ; - console.info("signed in as: " + username_li.textContent) ; + console.info("signed in as: " + username) ; } , USERNAME_LI_ID , WIKI_LOGIN)) ; } -function loadEditPage() +function LoadWikiData() +{ + LOG(WikiPages.length + " remaining") ; + + if (!!(WikiPage = WikiPages.shift())) IsReady = true ; else Done() ; +} + +function LoadEditPage() { - WikiPage = WikiPages.shift() ; var page_title = WikiPage[PACKAGE_NAME_KEY] ; - OpenUrl(EDIT_URL.replace(TITLE_URL_REGEX , page_title)) ; + GetUrl(EditPageUrl(page_title)) ; } -function submitEdit() +function SubmitEdit() { -if (DEBUG_VB) DBG("in") ; +if (DEBUG) DBG("in") ; var page_title = WikiPage[PACKAGE_NAME_KEY] ; var wiki_text = WikiPage[DESCRIPTION_KEY ] ; - QuitOnErr(Page.evaluate(function(wiki_text_input_id , edit_form_id , - wiki_text_regex , new_wiki_text) + if (Page.url != EditPageUrl(page_title)) { ForceQuit("edit page load failed") ; return ; } + + var post_wiki_text = EvalQuitOnErr(Page.evaluate(function(wiki_text_input_id , wiki_submit_button_id , + wiki_text_regex , new_wiki_text ) { - var wiki_text_input = document.getElementById(wiki_text_input_id) ; - var edit_form = wiki_text_input.form ; - var existing_wiki_text = wiki_text_input.value ; + var wiki_text_input = document.getElementById(wiki_text_input_id ) ; + var wiki_submit_button = document.getElementById(wiki_submit_button_id) ; + var existing_wiki_text = !!wiki_text_input && wiki_text_input.value ; + +// console.log("submitEdit() wiki_text_input=" + wiki_text_input ) ; +// console.log("submitEdit() wiki_text_input .nodeName" + wiki_text_input .nodeName) ; +// console.log("submitEdit() wiki_submit_button" + wiki_submit_button) ; +// console.log("submitEdit() wiki_submit_button.nodeName" + wiki_submit_button.nodeName) ; - if (!wiki_text_input || !edit_form) return "invalid edit page" ; + if (!wiki_text_input || wiki_text_input .nodeName != 'TEXTAREA' || + !wiki_submit_button || wiki_submit_button.nodeName != 'INPUT' ) + return "ERROR: invalid edit page" ; - var modified_wiki_text = existing_wiki_text.replace(wiki_text_regex , new_wiki_text) ; - var concatenated_wiki_text = existing_wiki_text + new_wiki_text ; + var modified_wiki_text = existing_wiki_text.replace(wiki_text_regex , new_wiki_text) ; + var concat_wiki_text = existing_wiki_text + new_wiki_text ; + var post_wiki_text = (!wiki_text_regex.test(existing_wiki_text)) ? concat_wiki_text : + (modified_wiki_text != existing_wiki_text ) ? modified_wiki_text : '' ; -console.log("submitEditPage() new_wiki_text=" + new_wiki_text) ; -console.log("submitEditPage() wiki_text_input.value IN=" + wiki_text_input.value) ; -console.log("submitEditPage() (modified_wiki_text == existing_wiki_text)=" + (modified_wiki_text == existing_wiki_text)) ; -console.log("submitEditPage() wiki_text_regex.test(existing_wiki_text)=" + wiki_text_regex.test(existing_wiki_text)) ; +// console.log("submitEdit() new_wiki_text=" + new_wiki_text) ; +// console.log("submitEdit() existing_wiki_text=" + existing_wiki_text) ; +// console.log("submitEdit() modified_wiki_text=" + modified_wiki_text) ; +// console.log("submitEdit() (modified_wiki_text == existing_wiki_text)=" + (modified_wiki_text == existing_wiki_text)) ; +// console.log("submitEdit() wiki_text_regex.test(existing_wiki_text)=" + wiki_text_regex.test(existing_wiki_text)) ; +// console.log("submitEdit() post_wiki_text=" + post_wiki_text) ; - if (!wiki_text_regex.test(existing_wiki_text)) wiki_text_input.value = concatenated_wiki_text ; - else if (modified_wiki_text != existing_wiki_text ) wiki_text_input.value = modified_wiki_text ; - else return '' ; + if (!!post_wiki_text) { wiki_text_input.value = post_wiki_text ; wiki_submit_button .click() ; } -console.log("submitEditPage() wiki_text_input.value OUT=" + wiki_text_input.value) ; + return post_wiki_text ; - edit_form.submit() ; + } , EDIT_TEXT_INPUT_ID , EDIT_SUBMIT_INPUT_ID , + WIKI_TEXT_REGEX , WIKI_TEXT_BEGIN + wiki_text + WIKI_TEXT_END)) ; - } , WIKITEXT_INPUT_ID , EDIT_FORM_ID , - WIKI_TEXT_REGEX , WIKI_TEXT_BEGIN + wiki_text + WIKI_TEXT_END)) ; +// DBG("returned post_wiki_text=" + post_wiki_text) ; -if (DEBUG_VB) DBG("out") ; + if (!post_wiki_text) { LOG("wiki text has not changed - skipping") ; NextPage() ; return ; } + +/* +console.log("edit_form.action=" + edit_form.action) ; +var edit_form_fields = edit_form.getElementsByTagName('input') ; +for (child_node_n in edit_form_fields) +{ + if (!edit_form.hasOwnProperty(child_node_n)) continue ; + + var child_node = edit_form_fields[child_node_n] ; + var field_key = child_node.name ; + var field_value = child_node.value ; + + if (child_node.type != 'hidden' || !field_key || !field_value) continue ; + +// field_key=wpStarttime field_value=20180121060903 +// field_key=wpEdittime field_value=20180121060903 +// field_key=wpAutoSummary field_value=d41d8cd98f00b204e9800998ecf8427e +// field_key=oldid field_value=0 +// field_key=parentRevId field_value=0 +// field_key=format field_value=text/x-wiki +// field_key=model field_value=wikitext +// field_key='wpEditToken field_value=d70645952b36fc2f29c73a665bb3acf45a642e80+\' +// field_key=wpUltimateParam field_value=1 +console.log("child_node=" + child_node + " field_key=" + field_key + " field_value=" + field_value) ; +} +*/ +/* + var xhr = new XMLHttpRequest() ; +var url = edit_form.action ; + var params = "lorem=ipsum&name=binny" ; + xhr.open('POST' , url , true) ; + xhr.setRequestHeader('Content-type' , 'application/x-www-form-urlencoded') ; + + xhr.onreadystatechange = function() + { + if (xhr.readyState == 4 && xhr.status == 200) + { +console.log(xhr.responseText) ; + } + }) ; + xhr.send(params) ; +*/ +/* + var post_body = 'wpTextbox1=' + post_wiki_text + + '&wpSave=' + 'Save page' + + '&wpEditToken=' + edit_token ; + PostUrl(edit_form_action , post_body) ; +*/ +if (DEBUG) DBG("out") ; } -function verifyEdit() +function VerifyEdit() { - QuitOnErr(Page.evaluate(function(wiki_text_input_id , edit_form_id , new_wiki_text) - { - var wiki_text_input = document.getElementById(wiki_text_input_id) ; - var existing_wiki_text = wiki_text_input.value ; - var wiki_text_regex = RegExp(new_wiki_text) ; +if (DEBUG) DBG("in") ; + + var page_title = WikiPage[PACKAGE_NAME_KEY] ; + var wiki_text = WikiPage[DESCRIPTION_KEY ] ; + var expected_url = WIKI_BASE_URL + '/' + page_title ; + page_title = page_title.replace(/_/g , ' ') ; + wiki_text = PARABOLA_ENTRY_HEADER.replace(/=/g , '').trim() + '\n' + wiki_text ; + +// DBG("Page.url=" + Page.url.toLowerCase()) ; DUMPFILE(Page.content) ; - if (!wiki_text_input || !wiki_text_regex.test(existing_wiki_text)) - return "edit failed" ; + if (Page.url.toLowerCase() != expected_url) { ForceQuit("edit post failed") ; return ; } - } , WIKITEXT_INPUT_ID , EDIT_FORM_ID , WIKI_TEXT_BEGIN + wiki_text + WIKI_TEXT_END)) ; + var err = EvalQuitOnErr(Page.evaluate(function(wiki_title_h1_id , wiki_text_div_id , + page_title , new_wiki_text ) + { + var wiki_title_h1 = document.getElementById(wiki_title_h1_id) ; + var wiki_text_div = document.getElementById(wiki_text_div_id) ; + var existing_wiki_title = !!wiki_title_h1 && wiki_title_h1.textContent.toLowerCase() ; + var existing_wiki_text = !!wiki_text_div && wiki_text_div.textContent.trim() ; + var wiki_text_regex = RegExp(new_wiki_text) ; + +console.log("wiki_title_h1=" + wiki_title_h1 + " wiki_title_h1.nodeName=" + wiki_title_h1.nodeName) ; +console.log("wiki_text_div=" + wiki_text_div + " wiki_text_div.nodeName=" + wiki_text_div.nodeName) ; +console.log("existing_wiki_title=" + existing_wiki_title) ; +console.log("page_title=" + page_title) ; +console.log("existing_wiki_text=" + existing_wiki_text) ; +console.log("new_wiki_text=" + new_wiki_text) ; +console.log("(existing_wiki_title == page_title )=" + (existing_wiki_title == page_title)) ; +console.log("wiki_text_regex.test(existing_wiki_text)=" + wiki_text_regex.test(existing_wiki_text)) ; + + if (!wiki_title_h1 || wiki_title_h1.nodeName != 'H1' || existing_wiki_title != page_title || + !wiki_text_div || wiki_text_div.nodeName != 'DIV' || !wiki_text_regex.test(existing_wiki_text)) + return "ERROR: edit post failed" ; + + } , WIKI_TITLE_H1_ID , WIKI_TEXT_DIV_ID , page_title , wiki_text)) ; + + if (!err) LOG("successfully updated: " + page_title) ; NextPage() ; } @@ -205,7 +307,7 @@ if (DEBUG_VB) DBG("ShouldQuit=" + ShouldQuit + " IsReady=" + IsReady) ; clearTimeout(StepTimeout) ; StepTimeout = setTimeout(function() { ForceQuit("timeout executing step") ; } , STEP_TIMEOUT) ; - var step_data = Steps.shift() ; + var step_data = Steps[StepN++] ; var step = step_data && step_data[STEP_FUNCTION_KEY] ; ShouldWait = step_data && step_data[PAGELOAD_WAIT_KEY] ; IsReady = false ; @@ -218,19 +320,37 @@ function PumpMainLoop() { setTimeout(MainLoop , 250) ; } function ForceQuit(quit_msg) { ShouldQuit = quit_msg ; PumpMainLoop() ; } -function Done() { LOG("done") ; phantom.exit() ; } +function Done() { LOG("done") ; phantom.exit() ; LOG('') ; } /* helpers */ -function OpenUrl(url) +function GetUrl(get_url) { -DBG("url=" + url) ; +DBG("get_url=" + get_url) ; - Page.open(url , function(status) { if (status != 'success') ForceQuit("status: " + status) ; }) ; + Page.open(get_url , function(status) { if (status != 'success') ForceQuit("status: " + status) ; }) ; } -function QuitOnErr(error_msg) { if (!!error_msg) ForceQuit(error_msg) ; else IsReady = !ShouldWait ; } +function PostUrl(post_url , post_body) +{ +DBG("post_url=" + post_url) ; + + Page.open(post_url , 'POST' , post_body , function(status) + { + if (status != 'success') ForceQuit("status: " + status) ; + }) ; +} + +function EvalQuitOnErr(result) +{ + if (typeof result == 'string' && !result.indexOf('ERROR:')) ForceQuit(result) ; + else IsReady = !ShouldWait ; + + return result ; +} + +function NextPage() { StepN = NEXT_PAGE_STEP ; IsReady = true ; } function IsA(an_object , expected_type) { @@ -243,6 +363,8 @@ function IsA(an_object , expected_type) return (Object.prototype.toString.call(an_object) === expected_type) ; } +function EditPageUrl(page_title) { return EDIT_URL.replace(TITLE_URL_REGEX , page_title) ; } + /* event hendlers */ @@ -257,7 +379,7 @@ Page.onUrlChanged = function() { ARGS.apply("Page.onUrlChang Page.onConsoleMessage = function(msg) { LOG(msg) ; } ; -var DEBUG = true ; var DEBUG_EVS = DEBUG && false ; var DEBUG_VB = DEBUG && false ; +var DEBUG = false ; var DEBUG_EVS = DEBUG && false ; var DEBUG_VB = DEBUG && false ; function LOG (log , color) { console.log((color || '\033[00;32m') + log + '\033[00m' ) ; } function LOG_STEP(name) { LOG("Step: " + name , '\033[01;32m') ; } function ERR (err) { LOG("ERROR: " + err , '\033[00;31m') ; } @@ -267,15 +389,19 @@ function DBG (dbg) { if (!DEBUG) return ; function ARGS () { if (!DEBUG_EVS) return ; LOG("EVENT: " + this) ; var args = arguments ; for (arg in args) LOG(" arg: " + JSON.stringify(args[arg])) ; } +function DUMPFILE(content) { require('fs').write('dump.txt' , content , 'w') ; } /* main entry */ -DefineStep(prepare , false) ; -DefineStep(loadLoginPage , true ) ; -DefineStep(submitLogin , true ) ; -DefineStep(verifyLogin , false) ; -DefineStep(loadEditPage , true ) ; -DefineStep(submitEdit , true ) ; -DefineStep(verifyEdit , false) ; +DefineStep(Prepare , false) ; +DefineStep(LoadLoginPage , true ) ; +DefineStep(SubmitLogin , true ) ; +DefineStep(VerifyLogin , false) ; +DefineStep(LoadWikiData , false) ; +DefineStep(LoadEditPage , true ) ; +DefineStep(SubmitEdit , true ) ; +DefineStep(VerifyEdit , false) ; +for (step_n in Steps) if (Steps[step_n][STEP_FUNCTION_KEY].name == 'LoadWikiData') NEXT_PAGE_STEP = step_n ; + MainLoop() ; |