87300988042f9b370f257fddf5a3ae0d21662851
galt
  Sat Feb 4 00:12:53 2017 -0800
Fixes for early warning during ajax callback; fixes for early warning in js. Changed to not only parse to but strip out the CSP header and js-with-nonce leaving cleaner html -- should create fewer "surprises" for existing screen-scraping code.

diff --git src/hg/js/utils.js src/hg/js/utils.js
index 2cfefd4..e31b9d6 100644
--- src/hg/js/utils.js
+++ src/hg/js/utils.js
@@ -662,46 +662,48 @@
             $(button).attr('title', 'Expand this '+titleDesc);
             contents.hide();
         } else {
             $(button).attr('title', 'Collapse this '+titleDesc);
             contents.show().trigger('show');
         }
         $(hidden).val(newVal);
         if (doAjax) {
             setCartVar(hiddenPrefix+"_"+prefix+"_close", newVal);
         }
         retval = false;
     }
     return retval;
 }
 
-function getNonce()
+function getNonce(debug)
 {   // Gets nonce value from page meta header
 var content = $("meta[http-equiv='Content-Security-Policy']").attr("content");
 if (!content)
     return "";
 // parse nonce like 'nonce-JDPiW8odQkiav4UCeXsa34ElFm7o'
 var sectionBegin = "'nonce-";
 var sectionEnd   = "'";
 var ix = content.indexOf(sectionBegin);
 if (ix < 0)
     return "";
 content = content.substring(ix+sectionBegin.length);
 ix = content.indexOf(sectionEnd);
 if (ix < 0)
     return "";
 content = content.substring(0,ix);
+if (debug)
+    alert('page nonce='+content);
 return content;
 }
 
 function warnBoxJsSetup()
 {   // Sets up warnBox if not already established.  This is duplicated from htmshell.c
     var html = "";
     html += "<center>";
     html += "<div id='warnBox' style='display:none;'>";
     html += "<CENTER><B id='warnHead'></B></CENTER>";
     html += "<UL id='warnList'></UL>";
     html += "<CENTER><button id='warnOK'></button></CENTER>";
     html += "</div></center>";
 
 
     // GALT TODO either add nonce or move the showWarnBox and hideWarnBox to some universal javascript 
@@ -1426,138 +1428,147 @@
     { // returns what falls between begToken and endToken as found in the string provided
     // Note ixBeg and ixEnd are optional bounds already established within string
         var bounds = bindings.inside(begToken,endToken,someString,ixBeg,ixEnd);
         if (bounds.start < bounds.stop)
             return someString.slice(bounds.start,bounds.stop);
 
         return '';
     }
 };
 
 function stripHgErrors(returnedHtml, whatWeDid)
 {   // strips HGERROR style 'early errors' and shows them in the warnBox
     // If whatWeDid !== null, we use it to return info about what we stripped out and
     // processed (current just warnMsg).
     var cleanHtml = returnedHtml;
+    var begToken = '<!-- HGERROR-START -->';
+    var endToken = '<!-- HGERROR-END -->';
     while (cleanHtml.length > 0) {
-        var bounds = bindings.outside('<!-- HGERROR-START -->','<!-- HGERROR-END -->',cleanHtml);
+        var bounds = bindings.outside(begToken,endToken,cleanHtml);
         if (bounds.start === -1)
             break;
-        var warnMsg = bindings.insideOut('<P>','</P>',cleanHtml,bounds.start,bounds.stop);
+        // OLD WAY var warnMsg = bindings.insideOut('<P>','</P>',cleanHtml,bounds.start,bounds.stop);
+	var warnMsg = cleanHtml.slice(bounds.start+begToken.length,bounds.stop-endToken.length);
         if (warnMsg.length > 0) {
             warn(warnMsg);
             if (whatWeDid)
                 whatWeDid.warnMsg = warnMsg;
         }
         cleanHtml = cleanHtml.slice(0,bounds.start) + cleanHtml.slice(bounds.stop);
     }
     return cleanHtml;
 }
 
 function stripJsFiles(returnedHtml, debug, whatWeDid)
 { // strips javascript files from html returned by ajax
     var cleanHtml = returnedHtml;
     var shlurpPattern=/<script type=\'text\/javascript\' SRC\=\'.*\'\><\/script\>/gi;
     if (debug || whatWeDid) {
         var jsFiles = cleanHtml.match(shlurpPattern);
         if (jsFiles && jsFiles.length > 0) {
 	    if (debug)
 		alert("jsFiles:'"+jsFiles+"'\n---------------\n"+cleanHtml); // warn() interprets html
 	    if (whatWeDid)
 		whatWeDid.jsFiles = jsFiles;
 	}
     }
     cleanHtml = cleanHtml.replace(shlurpPattern,"");
 
     return cleanHtml;
 }
 
-function stripCspHeader(returnedHtml, debug, whatWeDid)
+function stripCspHeader(html, debug, whatWeDid)
 { // strips CSP Header from html returned by ajax
-    var cleanHtml = returnedHtml;
     var shlurpPattern=/<meta http-equiv=\'Content-Security-Policy\' content=".*"\>/i;
     if (debug || whatWeDid) {
-        var csp = cleanHtml.match(shlurpPattern);
+        var csp = html.match(shlurpPattern);
         if (csp && csp.length > 0) {
 	    if (debug)
-		alert("csp:'"+csp+"'\n---------------\n"+cleanHtml); // warn() interprets html
+		alert("csp:'"+csp+"'\n---------------\n"+html); // warn() interprets html
 	    if (whatWeDid)
 		whatWeDid.csp = csp[0];
 	}
     }
-    cleanHtml = cleanHtml.replace(shlurpPattern,"");
-
-    return cleanHtml;
+    return html.replace(shlurpPattern,""); // Clean CSP meta tag.
 }
 
-function stripNonce(returnedHtml, debug)
-{ // strips nonce from returned ajax page
-    var cleanHtml = returnedHtml;
-    var csp = {};
-    stripCspHeader(returnedHtml, debug, csp);
-    var content = csp.csp;
+function parseNonce(content, debug)
+{ // parse nonce from returned ajax page csp header
+
     if (!content)
 	return "";
     // parse nonce like 'nonce-JDPiW8odQkiav4UCeXsa34ElFm7o'
     var sectionBegin = "'nonce-";
     var sectionEnd   = "'";
     var ix = content.indexOf(sectionBegin);
     if (ix < 0)
 	return "";
     content = content.substring(ix+sectionBegin.length);
     ix = content.indexOf(sectionEnd);
     if (ix < 0)
 	return "";
     content = content.substring(0,ix);
+    if (debug)
+	alert('ajax nonce='+content);
     return content;
 }
 
 function stripCssFiles(returnedHtml,debug)
 { // strips csst files from html returned by ajax
     var cleanHtml = returnedHtml;
     var shlurpPattern=/<LINK rel=\'STYLESHEET\' href\=\'.*\' TYPE=\'text\/css\' \/\>/gi;
     if (debug) {
         var cssFiles = cleanHtml.match(shlurpPattern);
         if (cssFiles && cssFiles.length > 0)
             alert("cssFiles:'"+cssFiles+"'\n---------------\n"+cleanHtml);
     }
     cleanHtml = cleanHtml.replace(shlurpPattern,"");
 
     return cleanHtml;
 }
 
-function stripJsNonce(returnedHtml, nonce, debug)
-{ // strips and returns embedded javascript from html returned by ajax with nonce
+function stripJsNonce(html, nonce, debug, whatWeDid)
+{ // Strips and returns embedded javascript from html returned by ajax with nonce
     var results=[];
-    var content = returnedHtml;
+    var content = "";
     var sectionBegin = "<script type='text/javascript' nonce='"+nonce+"'>";
     var sectionEnd   = "</script>";
     var lastIx = 0;
-    var ix = content.indexOf(sectionBegin, lastIx);
+    while (1) {
+	var ix = html.indexOf(sectionBegin, lastIx);
 	if (ix < 0)
-	return results;
-    ix += sectionBegin.length;
-    var ex = content.indexOf(sectionEnd, ix);
+	    break;
+	var ix2 = ix + sectionBegin.length;
+	var ex = html.indexOf(sectionEnd, ix2);
 	if (ex < 0)
-	return results;
-    var jsNonce = content.substring(ix,ex);
+	    break;
+	content += html.substring(lastIx,ix);
+	var jsNonce = html.substring(ix2,ex);
 	if (debug)
-	alert("jsNonce:'"+jsNonce);
+	    alert("jsNonce:"+jsNonce);
 	results.push(jsNonce);
-    lastIx = ex;
-    ex += sectionEnd.length;
-    return results;
+	lastIx = ex + sectionEnd.length;
+	}
+    // grab the last piece.
+    content += html.substring(lastIx);
+
+    //return results;
+    if (whatWeDid)
+	whatWeDid.js = results;
+
+    return content;
+
 }
 
 function charsAreHex(s)
 // are all the chars found hex?
 {
     var hexChars = "01234566789abcdefABCDEF";
     var d = false;
     var i = 0;
     if (s) {
 	d = true;
 	while (i < s.length) {
 	    if (hexChars.indexOf(s.charAt(i++)) < 0)
 		d = false;
 	}
     }
@@ -1598,30 +1609,69 @@
 	    if (!matched)
 		d = d + s.charAt(i++);
 	    }
     }
     return d;
 }
 
 
 function jsDecode(s)
 // For JS string values decode "\xHH" 
 {
     return nonAlphaNumericHexDecodeText(s, "\\x", "");
 }
 
 
+function stripCSPAndNonceJs(content,  debug, whatWeDid)
+// Strip CSP Header and script blocks with the ajax nonce.
+{
+
+    var pageNonce = getNonce(debug);
+
+    var csp = {};
+    content = stripCspHeader(content, debug, csp);
+
+    var ajaxNonce = parseNonce(csp.csp, debug);
+	
+    var jsBlocks = {};
+    content = stripJsNonce(content, ajaxNonce, debug, jsBlocks);
+
+    if (whatWeDid) {
+	whatWeDid.pageNonce = pageNonce;
+	whatWeDid.ajaxNonce = ajaxNonce;  // Not in use yet.
+	whatWeDid.js = jsBlocks.js;
+    }
+
+    return stripHgErrors(content, whatWeDid); // Certain early errors are not called via warnBox
+	
+}
+
+function appendNonceJsToPage(jsNonce)
+// Append ajax js blocks with nonce.
+// Create jsNonce by calling stripCSPAndNonceJs.
+// Call this after ajax html content has been added to the page/DOM.
+{
+    var i;
+    for (i=0; i<jsNonce.js.length; ++i) {
+	var sTag = document.createElement("script");
+	sTag.type = "text/javascript";
+	sTag.text = jsNonce.js[i];
+	sTag.setAttribute('nonce', jsNonce.pageNonce); // CSP2 Requires
+	document.head.appendChild(sTag);
+    }		
+}
+
 function stripJsEmbedded(returnedHtml, debug, whatWeDid)
 { 
   // GALT NOTE: this may have been mostly obsoleted by CSP2 changes.
   // There were 3 or 4 places in the code that even in production
   // had called this function stripJsEmbedded with debug=true, which means that
   // if any script tag blocks are present, they would be seen and shown
   // to the user.  This probably was because if these blocks were found
   // simply adding them to the div html from the ajax callback would result in 
   // their being ignored by the browser. It seems to be a security feature of browsers.
   // Meanwhile however inline event handlers in the html worked and were allowed.
   // So this was just a way to warn developers that their script blocks would have been ignored
   // and have no effect. I think this concern no longer applies after my CSP2 changes
   // because it is able to pull in all the js, whether from event handlers or what would
   // have been individual script blocks in the old days, and adds it to
   // the page with a nonce and appendChild.