[post_fsd_wiki.phantomjs]: add categories to wiki

2 files changed, 103 insertions, 60 deletions

diff --git a/blacklist-testdata.json b/blacklist-testdata.json
index a58da60..039c1e2 100644
--- a/blacklist-testdata.json
+++ b/blacklist-testdata.json
@@ -1,10 +1,14 @@
 [
   {
-    "original_package":  "dummy package 1"     ,
-    "short_description": "dummy description 1"
+    "original_package":  "dummy package 71"     ,
+    "libre_replacement": ""                     ,
+    "short_description": "dummy description 71" ,
+    "blacklist_tags":    [ "Category A" ]
   } ,
   {
-    "original_package":  "dummy package 2"     ,
-    "short_description": "dummy description 2"
+    "original_package":  "dummy package 72"           ,
+    "libre_replacement": "dummy replacement"          ,
+    "short_description": "dummy description 72"       ,
+    "blacklist_tags":    [ "Category A" , "nonfree" ]
   }
 ]
diff --git a/post_fsd_wiki.phantomjs b/post_fsd_wiki.phantomjs
index a6c3d4f..19c17f3 100644
--- a/post_fsd_wiki.phantomjs
+++ b/post_fsd_wiki.phantomjs
@@ -1,3 +1,14 @@
+/*\ post_fsd_wiki.phantomjs
+|*|
+|*|   this script is used to create/modify mediawiki pages on the FSD
+|*|
+|*|   one such wiki page for each blacklisted package
+|*|
+|*|   it must be executed by the phantomjs program
+|*|   and have access to a JSON file as created by the report.rb script
+\*/
+
+
 var BLACKLIST_DATA_FILE   = './blacklist-testdata.json' ;
 var WIKI_TEMPLATE_FILE    = './wiki-template.json' ;
 var WIKI_BASE_URL         = 'http://localhost/mediawiki/index.php' ;
@@ -9,7 +20,7 @@ var TITLE_URL_REGEX       = /_TITLE_/ ;
 var PARABOLA_ENTRY_HEADER = '== Parabola Blacklist Description ==\n' ;
 var WIKI_TEXT_BEGIN       = '<!-- PARABOLA BLACKLIST BEGIN (DO NOT EDIT) -->' + PARABOLA_ENTRY_HEADER ;
 var WIKI_TEXT_END         = '\n<!-- PARABOLA BLACKLIST END -->\n' ;
-var WIKI_TEXT_REGEX       = RegExp(WIKI_TEXT_BEGIN.replace(/\((.*)\)/ , '\\($1\\)') + '.*' + WIKI_TEXT_END) ;
+var WIKI_TEXT_REGEX       = RegExp(WIKI_TEXT_BEGIN.replace(/\((.*)\)/ , '\\($1\\)') + '(.|\n)*' + WIKI_TEXT_END) ;
 var LOGIN_IMPUT_ID        = 'wpName1' ;
 var PASS_IMPUT_ID         = 'wpPassword1' ;
 var LOGIN_SUBMIT_IMPUT_ID = 'wpLoginAttempt' ;
@@ -18,12 +29,22 @@ var EDIT_TEXT_INPUT_ID    = 'wpTextbox1' ;
 var EDIT_SUBMIT_INPUT_ID  = 'wpSave' ;
 var WIKI_TITLE_H1_ID      = 'firstHeading' ;
 var WIKI_TEXT_DIV_ID      = 'mw-content-text' ;
+var CATEGORIES_DIV_ID     = 'catlinks' ;
 var PACKAGE_NAME_KEY      = 'original_package' ;  // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[0]
 var REPLACEMENT_KEY       = 'libre_replacement' ; // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[1]
 var REFERENCE_KEY         = 'ref' ;               // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[2]
 var ENTRY_ID_KEY          = 'id' ;                // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[3]
 var DESCRIPTION_KEY       = 'short_description' ; // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[4]
 var BLACKLIST_TAGS_KEY    = 'blacklist_tags'      // BLACKLIST_DATA_FILE JSON
+var NONFREE_TAG           = 'nonfree' ;
+var SEMIFREE_TAG          = 'semifree' ;
+var USES_NONFREE_TAG      = 'uses-nonfree' ;
+var BRANDING_TAG          = 'branding' ;
+var TECHNICAL_TAG         = 'technical' ;
+var HAS_REPLACEMENT_TAG   = 'FIXME:package' ;
+var NEEDS_DESC_TAG        = 'FIXME:description' ;
+var ACCEPTED_TAGS         = [ NONFREE_TAG         , SEMIFREE_TAG   , USES_NONFREE_TAG ,
+                              HAS_REPLACEMENT_TAG , NEEDS_DESC_TAG                    ] ;
 var STEP_FUNCTION_KEY     = 'step-function' ;
 var PAGELOAD_WAIT_KEY     = 'wait-for-pageload' ;
 var STEP_TIMEOUT          = 3000 ; var StepTimeout ;
@@ -56,18 +77,30 @@ function Prepare()
 
   blacklist_data.forEach(function(package_data)
   {
-    var package_name = package_data[PACKAGE_NAME_KEY].trim() ;
-    var description  = package_data[DESCRIPTION_KEY ].trim() ;
-
-    if (!IsA(package_name , String) ||
-        !IsA(description  , String)  )
+    var package_name   = package_data[PACKAGE_NAME_KEY  ] && package_data[PACKAGE_NAME_KEY  ].trim() ;
+    var replacement    = package_data[REPLACEMENT_KEY   ] && package_data[REPLACEMENT_KEY   ].trim() ;
+    var description    = package_data[DESCRIPTION_KEY   ] && package_data[DESCRIPTION_KEY   ].trim() ;
+    var blacklist_tags = package_data[BLACKLIST_TAGS_KEY] ;
+
+    if ((!IsA(package_name   , String)                     ) ||
+        (!IsA(replacement    , String) && replacement != '') ||
+        (!IsA(description    , String)                     ) ||
+        (!IsA(blacklist_tags , Array )                     )  )
     { LOG_ERR("invalid package data - ignoring:\n" + JSON.stringify(package_data)) ; return ; }
 
-    LOG("found package: " + page_title) ;
+    package_name         = package_name.toLowerCase().replace(/ /g , '_') ;
+    var replacement_text = (!!replacement) ? '*Parabola Replacement: ' + replacement + '\n\n' : '' ;
+    blacklist_tags       = blacklist_tags.filter(function(tag) { return ~ACCEPTED_TAGS.indexOf(tag) ; })
+                                         .map   (function(tag) { return 'FSDG_' + tag                 }) ;
+    if (blacklist_tags.length == 0) { LOG(package_name + " tags are uninteresting - ignoring") ; return ; }
 
-    var wiki_page               = {} ;
-    wiki_page[PACKAGE_NAME_KEY] = package_name.toLowerCase().replace(/ /g , '_') ;
-    wiki_page[DESCRIPTION_KEY ] = description ;
+    LOG("loading package data: " + package_name) ;
+
+    var wiki_page                 = {} ;
+    wiki_page[PACKAGE_NAME_KEY  ] = package_name ;
+    wiki_page[REPLACEMENT_KEY   ] = replacement_text
+    wiki_page[DESCRIPTION_KEY   ] = description ;
+    wiki_page[BLACKLIST_TAGS_KEY] = blacklist_tags ;
     WikiPages.push(wiki_page) ;
   }) ;
 
@@ -76,19 +109,8 @@ function Prepare()
 
 function LoadLoginPage()
 {
-  if (WikiPages.length == 0) { ForceQuit("no WikiPages to consider") ; return ; }
-
-  for (page_n in WikiPages)
-  {
-    var wiki_page_data = WikiPages[page_n] ;
-    var page_title     = wiki_page_data[PACKAGE_NAME_KEY] ;
-    var page_text      = wiki_page_data[DESCRIPTION_KEY ] ;
-
-    if (!page_title || !IsA(page_title , String)) { ForceQuit("invalid wiki page data") ; return ; }
-    if (!page_text  || !IsA(page_text  , String)) { ForceQuit("invalid wiki page data") ; return ; }
-
-    LOG("loaded package data: " + page_title) ;
-  }
+  if (WikiPages.length > 0) { LOG(WikiPages.length + " WikiPages to consider") ; }
+  else                      { ForceQuit("no WikiPages to consider") ; return ;   }
 
   GetUrl(LOGIN_URL) ;
 }
@@ -151,8 +173,13 @@ function LoadEditPage()
 
 function SubmitEdit()
 {
-  var page_title = WikiPage[PACKAGE_NAME_KEY] ;
-  var wiki_text  = WikiPage[DESCRIPTION_KEY ] ;
+  var page_title       = WikiPage[PACKAGE_NAME_KEY  ] ;
+  var replacement_text = WikiPage[REPLACEMENT_KEY   ] ;
+  var wiki_text        = WikiPage[DESCRIPTION_KEY   ] ;
+  var wiki_categories  = WikiPage[BLACKLIST_TAGS_KEY] ;
+  wiki_text            = WIKI_TEXT_BEGIN + replacement_text + wiki_text + '\n\n' ;
+  wiki_categories.forEach(function(category) { wiki_text += '[[Category:' + category + ']]\n' }) ;
+  wiki_text           += WIKI_TEXT_END
 
   if (Page.url != EditPageUrl(page_title)) { ForceQuit("edit page load failed") ; return ; }
 
@@ -180,9 +207,7 @@ DBG('SubmitEditEvalOut')(new_wiki_text , existing_wiki_text , modified_wiki_text
     if (!!post_wiki_text) { wiki_text_input.value = post_wiki_text ; wiki_submit_button .click() ; }
 
     return post_wiki_text ;
-  } , EDIT_TEXT_INPUT_ID , EDIT_SUBMIT_INPUT_ID                                   ,
-      WIKI_TEXT_REGEX    , WIKI_TEXT_BEGIN + wiki_text + WIKI_TEXT_END , IsA , DBG)) ;
-
+  } , EDIT_TEXT_INPUT_ID , EDIT_SUBMIT_INPUT_ID , WIKI_TEXT_REGEX , wiki_text , IsA , DBG)) ;
 
 DBG('SubmitEditOut')(post_wiki_text) ;
 
@@ -191,34 +216,44 @@ DBG('SubmitEditOut')(post_wiki_text) ;
 
 function VerifyEdit()
 {
-  var page_title   = WikiPage[PACKAGE_NAME_KEY] ;
-  var wiki_text    = WikiPage[DESCRIPTION_KEY ] ;
-  var expected_url = WIKI_BASE_URL + '/' + page_title ;
-  page_title       = page_title.replace(/_/g , ' ') ;
-  wiki_text        = PARABOLA_ENTRY_HEADER.replace(/=/g , '').trim() + '\n' + wiki_text ;
+  var page_title       = WikiPage[PACKAGE_NAME_KEY  ] ;
+  var replacement_text = WikiPage[REPLACEMENT_KEY   ] ;
+  var wiki_text        = WikiPage[DESCRIPTION_KEY   ] ;
+  var wiki_categories  = WikiPage[BLACKLIST_TAGS_KEY] ;
+  var expected_url     = WIKI_BASE_URL + '/' + page_title ;
+  page_title           = page_title              .replace(/_/g , ' ') ;
+  wiki_text            = PARABOLA_ENTRY_HEADER   .replace(/=/g  , '').trim() + '\n'             +
+                         replacement_text        .replace(/\*/g , '').trim() + '\n' + wiki_text ;
+  var categories       = wiki_categories.join('').replace(/_/g , ' ') ;
+  var categories_text  = ((wiki_categories.length > 1) ? 'Categories: ' : 'Category: ') + categories ;
 
 if (DEBUG_VB) DUMPFILE() ;
 
-  if (Page.url.toLowerCase() != expected_url) { ForceQuit("edit post failed") ; return ; }
+  if (Page.url.toLowerCase() != expected_url) { ForceQuit("edit post expected_url failed") ; return ; }
 
-  var err = EvalQuitOnErr(Page.evaluate(function(wiki_title_h1_id , wiki_text_div_id ,
-                                                 page_title       , new_wiki_text    ,
-                                                 IsA              , DBG              )
+  var err = EvalQuitOnErr(Page.evaluate(function(wiki_title_h1_id , wiki_text_div_id , categories_div_id ,
+                                                 page_title       , new_wiki_text    , new_categories    ,
+                                                 IsA              , DBG                                  )
   {
-    var wiki_title_h1       = document.getElementById(wiki_title_h1_id) ;
-    var wiki_text_div       = document.getElementById(wiki_text_div_id) ;
-    var existing_wiki_title = !!wiki_title_h1 && wiki_title_h1.textContent.toLowerCase() ;
-    var existing_wiki_text  = !!wiki_text_div && wiki_text_div.textContent.trim() ;
-    var wiki_text_regex     = RegExp(new_wiki_text) ;
-
-DBG('VerifyEdit')(wiki_title_h1 , wiki_text_div , existing_wiki_title , page_title , existing_wiki_text , new_wiki_text , wiki_text_regex) ;
-
-    if (!IsA(wiki_title_h1  , 'H1' ) || existing_wiki_title != page_title         ||
-        !IsA(wiki_text_div  , 'DIV') || !wiki_text_regex .test(existing_wiki_text ))
-      return "ERROR: edit post failed" ;
+    var wiki_title_h1       = document.getElementById(wiki_title_h1_id ) ;
+    var wiki_text_div       = document.getElementById(wiki_text_div_id ) ;
+    var categories_div      = document.getElementById(categories_div_id) ;
+    var existing_wiki_title = !!wiki_title_h1  && wiki_title_h1 .textContent.toLowerCase() ;
+    var existing_wiki_text  = !!wiki_text_div  && wiki_text_div .textContent.trim() ;
+    var existing_categories = !!categories_div && categories_div.textContent.trim() ;
+    var wiki_text_regex     = RegExp(new_wiki_text ) ;
+    var categories_regex    = RegExp(new_categories) ;
+
+DBG('VerifyEdit')(wiki_title_h1 , wiki_text_div , categories_div , existing_wiki_title , page_title , existing_wiki_text , new_wiki_text , existing_categories , new_categories , wiki_text_regex , categories_regex) ;
+
+    if (!IsA(wiki_title_h1  , 'H1' ) || existing_wiki_title != page_title           ||
+        !IsA(wiki_text_div  , 'DIV') || !wiki_text_regex .test(existing_wiki_text ) ||
+        !IsA(categories_div , 'DIV') || !categories_regex.test(existing_categories)  )
+      return "ERROR: edit post data failed" ;
 
     return '' ;
-  } , WIKI_TITLE_H1_ID , WIKI_TEXT_DIV_ID , page_title , wiki_text , IsA , DBG)) ;
+  } , WIKI_TITLE_H1_ID , WIKI_TEXT_DIV_ID , CATEGORIES_DIV_ID            ,
+      page_title       , wiki_text        , categories_text   , IsA , DBG)) ;
 
   if (!err) LOG("successfully updated: " + page_title) ; NextPage() ;
 }
@@ -319,7 +354,7 @@ Page.onUrlChanged   = function() {                   LOG_ARGS.apply("Page.onUrlC
 
 Page.onConsoleMessage = function(msg) { DBG('')(msg) ; } ;
 
-var DEBUG = false ; var DEBUG_EVS = DEBUG && false ; var DEBUG_VB = DEBUG && false ;
+var DEBUG = true ; var DEBUG_EVS = DEBUG && false ; var DEBUG_VB = DEBUG && false ;
 function LOG     (log , color) { console.log((color || '\033[00;32m') + log  + '\033[00m'   ) ; }
 function LOG_STEP(name)        { LOG("Step: "                         + name , '\033[01;32m') ; }
 function LOG_ERR (err)         { LOG("ERROR: "                        + err  , '\033[00;31m') ; }
@@ -345,17 +380,21 @@ function DBG(context)
     Dbg("SubmitEdit() modified_wiki_text="                         + modified_wiki_text                        ) ;
     Dbg("SubmitEdit() post_wiki_text="                             + post_wiki_text                            ) ;
     Dbg("SubmitEdit() (modified_wiki_text == existing_wiki_text)=" + (modified_wiki_text == existing_wiki_text)) ;
-    Dbg("SubmitEdit() wiki_text_regex.test(existing_wiki_text)="   + wiki_text_regex.test(existing_wiki_text)  ) ; }
+    Dbg("SubmitEdit() wiki_text_regex.test(existing_wiki_text  )=" + wiki_text_regex.test(existing_wiki_text)  ) ; }
   function DbgSubmitEditOut(post_wiki_text) { if (DEBUG_VB) Dbg("SubmitEdit() returned post_wiki_text=" + post_wiki_text) ; }
-  function DbgVerifyEdit(wiki_title_h1 , wiki_text_div , existing_wiki_title , page_title , existing_wiki_text , new_wiki_text , wiki_text_regex)
-  { Dbg("VerifyEdit() wiki_title_h1="                              + wiki_title_h1  + " wiki_title_h1.nodeName="  + wiki_title_h1.nodeName ) ;
-    Dbg("VerifyEdit() wiki_text_div="                              + wiki_text_div  + " wiki_text_div.nodeName="  + wiki_text_div.nodeName ) ;
+  function DbgVerifyEdit(wiki_title_h1 , wiki_text_div , categories_div , existing_wiki_title , page_title , existing_wiki_text , new_wiki_text , existing_categories , new_categories , wiki_text_regex , categories_regex)
+  { Dbg("VerifyEdit() wiki_title_h1="                              + wiki_title_h1  + " wiki_title_h1 .nodeName=" + wiki_title_h1.nodeName ) ;
+    Dbg("VerifyEdit() wiki_text_div="                              + wiki_text_div  + " wiki_text_div .nodeName=" + wiki_text_div.nodeName ) ;
+    Dbg("VerifyEdit() categories_div="                             + categories_div + " categories_div.nodeName=" + categories_div.nodeName) ;
     Dbg("VerifyEdit() existing_wiki_title="                        + existing_wiki_title                                                   ) ;
     Dbg("VerifyEdit() page_title="                                 + page_title                                                            ) ;
     Dbg("VerifyEdit() existing_wiki_text="                         + existing_wiki_text                                                    ) ;
     Dbg("VerifyEdit() new_wiki_text="                              + new_wiki_text                                                         ) ;
-    Dbg("VerifyEdit() (existing_wiki_title == page_title    )="    + (existing_wiki_title == page_title)                                   ) ;
-    Dbg("VerifyEdit() wiki_text_regex .test(existing_wiki_text)="  + wiki_text_regex .test(existing_wiki_text)                             ) ; }
+    Dbg("VerifyEdit() existing_categories="                        + existing_categories                                                   ) ;
+    Dbg("VerifyEdit() new_categories="                             + new_categories                                                        ) ;
+    Dbg("VerifyEdit() (existing_wiki_title == page_title       )=" + (existing_wiki_title == page_title)                                   ) ;
+    Dbg("VerifyEdit() wiki_text_regex .test(existing_wiki_text )=" + wiki_text_regex .test(existing_wiki_text)                             ) ;
+    Dbg("VerifyEdit() categories_regex.test(existing_categories)=" + categories_regex.test(existing_categories)                            ) ; }
   function DbgMainLoop() { if (DEBUG_VB) Dbg("MainLoop() ShouldQuit=" + ShouldQuit + " IsReady=" + IsReady) ; }
   function DbgGetUrl(get_url) { Dbg("GetUrl() " + get_url) ; }