353a34ac7e7638457db3f55e57452069113d860b braney Sun Sep 6 13:34:28 2026 -0700 Add a bigNet track type, a net of alignments in a bigBed, refs #20824 Track hubs have had no way to show a real net. The usual stand-in is a net rendered as a maf, which loses the level structure that makes a net useful for establishing orthologous sequence. bigNet holds the netAlign columns in a bigBed, so a hub can carry the net itself. The format is bed6+20: the target in chrom/chromStart/chromEnd, the query sequence in name, the query strand in strand, then level and the rest of the netAlign fields. The trackDb line is type bigNet <targetDb> <chainTrack> mirroring type netAlign. chainTrack is the plain trackDb name of the bigChain track in the same hub; hgc adds the hub prefix itself. chainNetLoadRangeHub() builds a chainNet from a bigBed range query and hands it to the same helpToNet() the SQL path uses, so the nesting is rebuilt the same way. netDraw picks its loader off tg->isBigBed and the drawing code below that is untouched. genericNetClick does the same for the details page and follows the named chain track for the alignment. Also bounds the level walk in helpToNet() by help->maxDepth. It could read one past the end of the levels array. netToBigNet converts a net file to bedToBigBed input. It writes the tab line itself rather than calling bigNetTabOut, because autoSql prints a double with %g and that drops digits off a chain score. diff --git src/hg/hubApi/apiUtils.c src/hg/hubApi/apiUtils.c index 2d2ac929b97..e1f8c244c4f 100644 --- src/hg/hubApi/apiUtils.c +++ src/hg/hubApi/apiUtils.c @@ -1,1067 +1,1068 @@ /* utility functions for data API business */ #include "trackHub.h" #include "dataApi.h" #include "net.h" #include "wikiLink.h" /* when measureTiming is used */ static long processingStart = 0; void startProcessTiming() /* for measureTiming, beginning processing */ { processingStart = clock1000(); } void apiFinishOutput(int errorCode, char *errorString, struct jsonWrite *jw) /* finish json output, potential output an error code other than 200 */ { /* this is the first time any output to stdout has taken place for * json output, therefore, start with the appropriate header. */ puts("X-Content-Type-Options: nosniff"); puts("Content-Type:application/json"); /* potentially with an error code return in the header */ if (errorCode) { char errString[2048]; safef(errString, sizeof(errString), "Status: %d %s",errorCode,errorString); puts(errString); if (err429 == errorCode) puts("Retry-After: 30"); } else if (reachedMaxItems) { char errString[2048]; safef(errString, sizeof(errString), "Status: %d %s",err206,err206Msg); puts(errString); } puts("\n"); if (measureTiming) { long nowTime = clock1000(); long long et = nowTime - processingStart; jsonWriteNumber(jw, "totalTimeMs", et); } if (itemsReturned) jsonWriteNumber(jw, "itemsReturned", itemsReturned); if (reachedMaxItems) { jsonWriteBoolean(jw, "maxItemsLimit", TRUE); if (downloadUrl && downloadUrl->string) jsonWriteString(jw, "dataDownloadUrl", downloadUrl->string); } jsonWriteObjectEnd(jw); fputs(jw->dy->string,stdout); } /* void apiFinishOutput(int errorCode, char *errorString, ... ) */ void apiErrAbort(int errorCode, char *errString, char *format, ...) /* Issue an error message in json format, and exit(0) */ { char errMsg[2048]; va_list args; va_start(args, format); vsnprintf(errMsg, sizeof(errMsg), format, args); struct jsonWrite *jw = apiStartOutput(); jsonWriteString(jw, "error", errMsg); jsonWriteNumber(jw, "statusCode", errorCode); jsonWriteString(jw, "statusMessage", errString); apiFinishOutput(errorCode, errString, jw); cgiExitTime("hubApi err", enteredMainTime); exit(0); } struct jsonWrite *apiStartOutput() /* begin json output with standard header information for all requests */ { time_t timeNow = time(NULL); // struct tm tm; // gmtime_r(&timeNow, &tm); struct jsonWrite *jw = jsonWriteNew(); jsonWriteObjectStart(jw, NULL); // not recommended: jsonWriteString(jw, "apiVersion", "v"CGI_VERSION); // not needed jsonWriteString(jw, "source", "UCSantaCruz"); jsonWriteDateFromUnix(jw, "downloadTime", (long long) timeNow); jsonWriteNumber(jw, "downloadTimeStamp", (long long) timeNow); return jw; } /* these json type strings beyond 'null' are types for UCSC jsonWrite() * functions. The real 'official' json types are just the first six */ char *jsonTypeStrings[] = { "string", /* type 0 */ "number", /* type 1 */ "object", /* type 2 */ "array", /* type 3 */ "boolean", /* type 4 */ "null", /* type 5 */ "double" /* type 6 */ }; /* this set of SQL types was taken from a survey of all the table * descriptions on all tables on hgwdev. This is not necessarily * a full range of all SQL types possible, just what we have used in * UCSC databases. The fallback default will be string. * bigint binary bit blob char date datetime decimal double enum float int * longblob longtext mediumblob mediumint mediumtext set smallint text * timestamp tinyblob tinyint tinytext unsigned varbinary varchar * Many of these are not yet encountered since they are often in * non-positional tables. This system doesn't yet output non-positional * tables. */ static int sqlTypeToJsonType(char *sqlType) /* convert SQL data type to JSON data type (index into jsonTypes[]) */ { /* assume string, good enough for just about anything */ int typeIndex = JSON_STRING; if (startsWith("tinyint(1)", sqlType)) typeIndex = JSON_BOOLEAN; else if (startsWith("bigint", sqlType) || startsWith("int", sqlType) || startsWith("mediumint", sqlType) || startsWith("smallint", sqlType) || startsWith("tinyint", sqlType) || startsWith("unsigned", sqlType) ) typeIndex = JSON_NUMBER; else if (startsWith("decimal", sqlType) || startsWith("double", sqlType) || startsWith("float", sqlType) ) typeIndex = JSON_DOUBLE; else if (startsWith("binary", sqlType) || startsWith("bit", sqlType) || startsWith("blob", sqlType) || startsWith("char", sqlType) || startsWith("date", sqlType) || startsWith("datetime", sqlType) || startsWith("enum", sqlType) || startsWith("longblob", sqlType) || startsWith("longtext", sqlType) || startsWith("mediumblob", sqlType) || startsWith("set", sqlType) || startsWith("text", sqlType) || startsWith("time", sqlType) || startsWith("timestamp", sqlType) || startsWith("tinyblob", sqlType) || startsWith("tinytext", sqlType) || startsWith("varbinary", sqlType) || startsWith("varchar", sqlType) ) typeIndex = JSON_STRING; return typeIndex; } /* static int sqlTypeToJsonType(char *sqlType) */ int autoSqlToJsonType(char *asType) /* convert an autoSql field type to a Json type */ { /* assume string, good enough for just about anything */ int typeIndex = JSON_STRING; if (startsWith("int", asType) || startsWith("uint", asType) || startsWith("short", asType) || startsWith("ushort", asType) || startsWith("byte", asType) || startsWith("ubyte", asType) || startsWith("bigit", asType) ) typeIndex = JSON_NUMBER; else if (startsWith("float", asType)) typeIndex = JSON_DOUBLE; else if (startsWith("char", asType) || startsWith("string", asType) || startsWith("lstring", asType) || startsWith("enum", asType) || startsWith("set", asType) ) typeIndex = JSON_STRING; return typeIndex; } /* int asToJsonType(char *asType) */ /* temporarily from table browser until proven works, then move to library */ /* UNFORTUNATELY, this version needs an extra argument: tdb * the table browser has a global hash to satisify that requirement */ struct asObject *asForTable(struct sqlConnection *conn, char *table, struct trackDb *tdb) /* Get autoSQL description if any associated with table. */ /* Wrap some error catching around asForTable. */ { if (tdb != NULL) return asForTdb(conn,tdb); // Some cases are for tables with no tdb! struct asObject *asObj = NULL; if (sqlTableExists(conn, "tableDescriptions")) { struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) { char query[256]; sqlSafef(query, sizeof(query), "select autoSqlDef from tableDescriptions where tableName='%s'", table); char *asText = asText = sqlQuickString(conn, query); // If no result try split table. (not likely) if (asText == NULL) { sqlSafef(query, sizeof(query), "select autoSqlDef from tableDescriptions where tableName='chrN_%s'", table); asText = sqlQuickString(conn, query); } if (asText != NULL && asText[0] != 0) { asObj = asParseText(asText); } freez(&asText); } errCatchEnd(errCatch); errCatchFree(&errCatch); } return asObj; } int tableColumns(struct sqlConnection *conn, char *table, char ***nameReturn, char ***typeReturn, int **jsonTypes) /* return the column names, and their MySQL data type, for the given table * return number of columns (aka 'fields') */ { struct sqlFieldInfo *fi, *fiList = sqlFieldInfoGet(conn, table); int columnCount = slCount(fiList); char **namesReturn = NULL; char **typesReturn = NULL; int *jsonReturn = NULL; AllocArray(namesReturn, columnCount); AllocArray(typesReturn, columnCount); AllocArray(jsonReturn, columnCount); int i = 0; for (fi = fiList; fi; fi = fi->next) { namesReturn[i] = cloneString(fi->field); typesReturn[i] = cloneString(fi->type); jsonReturn[i] = sqlTypeToJsonType(fi->type); i++; } *nameReturn = namesReturn; *typeReturn = typesReturn; *jsonTypes = jsonReturn; return columnCount; } struct trackHub *errCatchTrackHubOpen(char *hubUrl) /* use errCatch around a trackHub open in case it fails */ { struct trackHub *hub = NULL; struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) { char *errMessage; unsigned hubId = hubFindOrAddUrlInStatusTable(NULL, hubUrl, &errMessage); /* this hubFromIdNoAbort() call will update the hub status error message * if it has recovered from previous errors. */ struct hubConnectStatus *hubStatus = hubFromIdNoAbort(hubId); // if we got an error, throw error if (NULL == hubStatus) /* this should not happen */ errAbort("could not find hub %u in status table", hubId); if (!isEmpty(hubStatus->errorMessage)) { stripChar(hubStatus->errorMessage, '\n'); errAbort("%s", hubStatus->errorMessage); } // use hubId in hubName char buffer[4096]; safef(buffer, sizeof buffer, "hub_%d", hubId); hub = trackHubOpen(hubUrl, buffer); } errCatchEnd(errCatch); if (errCatch->gotError) { apiErrAbort(err404, err404Msg, "error opening hubUrl: '%s', '%s'", hubUrl, errCatch->message->string); } errCatchFree(&errCatch); return hub; } static int trackDbTrackCmp(const void *va, const void *vb) /* Compare to sort based on 'track' name; use shortLabel as secondary sort key. * Note: parallel code to hgTracks.c:tgCmpPriority */ { const struct trackDb *a = *((struct trackDb **)va); const struct trackDb *b = *((struct trackDb **)vb); int dif = strcmp(a->track, b->track); if (dif < 0) return -1; else if (dif == 0.0) return strcasecmp(a->shortLabel, b->shortLabel); else return 1; } struct trackDb *obtainTdb(struct trackHubGenome *genome, char *db) /* return a full trackDb given the hub genome pointer, or ucsc database name */ { struct trackDb *tdb = NULL; if (db) tdb = hTrackDb(db); else tdb = trackHubAddTracksGenome(genome); /* remove the synthetic gcOnFly track it exists only for hgTracks display */ struct trackDb *filtered = NULL; struct trackDb *next; while (tdb) { next = tdb->next; if (sameString(GC_ON_FLY_TRACK_NAME, trackHubSkipHubName(tdb->track))) { tdb->next = NULL; } else { slAddHead(&filtered, tdb); } tdb = next; } slReverse(&filtered); slSort(filtered, trackDbTrackCmp); return filtered; } struct trackDb *findTrackDb(char *track, struct trackDb *tdb) /* search tdb structure for specific track, recursion on subtracks */ { struct trackDb *trackFound = NULL; for (trackFound = tdb; trackFound; trackFound = trackFound->next) { if (trackFound->subtracks) { struct trackDb *subTrack = findTrackDb(track, trackFound->subtracks); if (subTrack) { if (sameOk(trackHubSkipHubName(subTrack->track), track)) trackFound = subTrack; } } if (sameOk(trackHubSkipHubName(trackFound->track), track)) break; } return trackFound; } boolean allowedBigBedType(char *type) /* return TRUE if the big* bed-like type is to be supported * add to this list as the big* bed-like supported types are expanded */ { if (startsWithWord("bigBarChart", type) || startsWithWord("bigBed", type) || startsWithWord("bigGenePred", type) || startsWithWord("bigInteract", type) || startsWithWord("bigLolly", type) || startsWithWord("bigRmsk", type) || startsWithWord("bigDbSnp", type) || startsWithWord("bigMaf", type) || startsWithWord("bigChain", type) || + startsWithWord("bigNet", type) || startsWithWord("bigPsl", type) ) return TRUE; else return FALSE; } struct bbiFile *bigFileOpen(char *trackType, char *bigDataUrl) /* open bigDataUrl for correct trackType and error catch if failure */ { struct bbiFile *bbi = NULL; struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) { if (allowedBigBedType(trackType)) bbi = bigBedFileOpenAlias(bigDataUrl, chromAliasFindAliases); else if (startsWith("bigWig", trackType)) bbi = bigWigFileOpenAlias(bigDataUrl, chromAliasFindAliases); } errCatchEnd(errCatch); if (errCatch->gotError) { apiErrAbort(err404, err404Msg, "error opening bigFile URL: '%s', '%s'", bigDataUrl, errCatch->message->string); } errCatchFree(&errCatch); return bbi; } int chromInfoCmp(const void *va, const void *vb) /* Compare to sort based on size */ { const struct chromInfo *a = *((struct chromInfo **)va); const struct chromInfo *b = *((struct chromInfo **)vb); int dif; dif = (long) a->size - (long) b->size; return dif; } struct trackHubGenome *findHubGenome(struct trackHub *hub, char *genome, char *endpoint, char *hubUrl) /* given open 'hub', find the specified 'genome' called from 'endpoint' */ { struct trackHubGenome *hubGenome = NULL; for (hubGenome = hub->genomeList; hubGenome; hubGenome = hubGenome->next) { if (sameString(genome, trackHubSkipHubName(hubGenome->name))) break; } if (NULL == hubGenome) apiErrAbort(err400, err400Msg, "failed to find specified genome=%s for endpoint '%s' given hubUrl '%s'", genome, endpoint, hubUrl); return hubGenome; } char *verifyLegalArgs(char *validArgList[]) /* validArgList is an array of strings for valid arguments * returning string of any other arguments not on that list found in * cgiVarList(), NULL when none found. */ { struct hash *validHash = NULL; char *words[32]; int wordCount = 0; int i = 0; for ( ; isNotEmpty(validArgList[i]); ++i ) { ++wordCount; words[i] = validArgList[i]; } if (wordCount) { validHash = hashNew(0); int i; for (i = 0; i < wordCount; ++i) hashAddInt(validHash, words[i], 1); } int extrasFound = 0; struct dyString *extras = dyStringNew(128); struct cgiVar *varList = cgiVarList(); struct cgiVar *var = varList; for ( ; var; var = var->next) { if (sameWord("cgiSpoof", var->name)) continue; if (sameWord("debug", var->name)) // debug silent tolerated, does nothing continue; if (sameWord("cmd", var->name)) continue; if (sameWord("measureTiming", var->name)) continue; if (NULL == validHash) { dyStringPrintf(extras, ";%s=%s", var->name, var->val); ++extrasFound; } else if (0 == hashIntValDefault(validHash, var->name, 0)) { if (extrasFound) dyStringPrintf(extras, ";%s=%s", var->name, var->val); else dyStringPrintf(extras, "%s=%s", var->name, var->val); ++extrasFound; } } if (extrasFound) return dyStringCannibalize(&extras); else return NULL; } static int genArkCmpName(const void *va, const void *vb) /* Compare two genark elements: gcAccession, ignore case. */ { const struct genark *a = *((struct genark **)va); const struct genark *b = *((struct genark **)vb); return strcasecmp(a->gcAccession, b->gcAccession); } static int dbDbCmpName(const void *va, const void *vb) /* Compare two dbDb elements: name, ignore case. */ { const struct dbDb *a = *((struct dbDb **)va); const struct dbDb *b = *((struct dbDb **)vb); return strcasecmp(a->name, b->name); } long long genArkSize() /* return the number of rows in genark table */ { char query[1024]; struct sqlConnection *conn = hConnectCentral(); sqlSafef(query, sizeof(query), "SELECT COUNT(*) FROM %s", genarkTableName()); return sqlQuickLongLong(conn, query); } struct genark *genArkList(char *oneAccession) /* return the genark table as an slList, or just the one accession when given */ { char query[1024]; struct sqlConnection *conn = hConnectCentral(); if (oneAccession) sqlSafef(query, sizeof(query), "SELECT * FROM %s WHERE gcAccession = '%s'", genarkTableName(), oneAccession); else sqlSafef(query, sizeof(query), "SELECT * FROM %s ORDER BY gcAccession limit %d", genarkTableName(), maxItemsOutput); struct genark *list = NULL, *el = NULL; struct sqlResult *sr = sqlGetResult(conn, query); char **row; while ((row = sqlNextRow(sr)) != NULL) { el = genarkLoad(row); slAddHead(&list, el); } sqlFreeResult(&sr); hDisconnectCentral(&conn); slSort(&list, genArkCmpName); return list; } struct dbDb *ucscDbDb() /* return the dbDb table as an slList */ { char query[1024]; struct sqlConnection *conn = hConnectCentral(); char *showActive0 = cfgOptionDefault("hubApi.showActive0", "off"); if (sameWord("on", showActive0)) sqlSafef(query, sizeof(query), "select * from dbDb"); else sqlSafef(query, sizeof(query), "select * from dbDb where active=1"); struct dbDb *dbList = NULL, *el = NULL; struct sqlResult *sr = sqlGetResult(conn, query); char **row; while ((row = sqlNextRow(sr)) != NULL) { el = dbDbLoad(row); slAddHead(&dbList, el); } sqlFreeResult(&sr); hDisconnectCentral(&conn); slSort(&dbList, dbDbCmpName); return dbList; } boolean isSupportedType(char *type) /* is given type in the supportedTypes list ? */ { boolean ret = FALSE; struct slName *el; for (el = supportedTypes; el; el = el->next) { if (startsWith(el->name, type)) { ret = TRUE; break; } } return ret; } void wigColumnTypes(struct jsonWrite *jw, char *track) /* output column headers for a wiggle data output schema */ { jsonWriteListStart(jw, track); jsonWriteObjectStart(jw, NULL); jsonWriteString(jw, "name", "chrom"); jsonWriteString(jw, "sqlType", "varchar"); jsonWriteString(jw, "jsonType", "string"); jsonWriteString(jw, "description", "chromosome name"); jsonWriteObjectEnd(jw); jsonWriteObjectStart(jw, NULL); jsonWriteString(jw, "name", "start"); jsonWriteString(jw, "sqlType", "int"); jsonWriteString(jw, "jsonType", "number"); jsonWriteString(jw, "description", "chromStart: 0-based chromosome start position"); jsonWriteObjectEnd(jw); jsonWriteObjectStart(jw, NULL); jsonWriteString(jw, "name", "end"); jsonWriteString(jw, "sqlType", "int"); jsonWriteString(jw, "jsonType", "number"); jsonWriteString(jw, "description", "chromEnd: 1-based chromosome end position"); jsonWriteObjectEnd(jw); jsonWriteObjectStart(jw, NULL); jsonWriteString(jw, "name", "value"); jsonWriteString(jw, "sqlType", "float"); jsonWriteString(jw, "jsonType", "number"); jsonWriteString(jw, "description", "numerical data value for this location:start-end"); jsonWriteObjectEnd(jw); jsonWriteListEnd(jw); } /* void wigColumnTypes(struct jsonWrite jw) */ void outputSchema(struct trackDb *tdb, struct jsonWrite *jw, char *columnNames[], char *columnTypes[], int jsonTypes[], struct hTableInfo *hti, int columnCount, int asColumnCount, struct asColumn *columnEl, char *keyStr) /* print out the SQL schema for this trackDb */ { if (tdb && isWiggleDataTable(tdb->type)) { wigColumnTypes(jw, keyStr); } else { jsonWriteListStart(jw, keyStr); int i = 0; for (i = 0; i < columnCount; ++i) { jsonWriteObjectStart(jw, NULL); jsonWriteString(jw, "name", columnNames[i]); jsonWriteString(jw, "sqlType", columnTypes[i]); jsonWriteString(jw, "jsonType", jsonTypeStrings[jsonTypes[i]]); if ((0 == i) && (hti && hti->hasBin)) jsonWriteString(jw, "description", "Indexing field to speed chromosome range queries"); else if (columnEl && isNotEmpty(columnEl->comment)) jsonWriteString(jw, "description", columnEl->comment); else jsonWriteString(jw, "description", ""); /* perhaps move the comment pointer forward */ if (columnEl) { if (asColumnCount == columnCount) columnEl = columnEl->next; else if (! ((0 == i) && (hti && hti->hasBin))) columnEl = columnEl->next; } jsonWriteObjectEnd(jw); } jsonWriteListEnd(jw); } } void bigColumnTypes(struct jsonWrite *jw, struct sqlFieldType *fiList, struct asObject *as, char *track) /* show the column types from a big file autoSql definitions */ { struct asColumn *columnEl = as->columnList; jsonWriteListStart(jw, track); struct sqlFieldType *fi = fiList; for ( ; fi; fi = fi->next, columnEl = columnEl->next) { int jsonType = autoSqlToJsonType(fi->type); jsonWriteObjectStart(jw, NULL); jsonWriteString(jw, "name", fi->name); jsonWriteString(jw, "sqlType", fi->type); jsonWriteString(jw, "jsonType",jsonTypeStrings[jsonType]); if (columnEl && isNotEmpty(columnEl->comment)) jsonWriteString(jw, "description", columnEl->comment); else jsonWriteString(jw, "description", ""); jsonWriteObjectEnd(jw); } jsonWriteListEnd(jw); } boolean trackHasData(struct trackDb *tdb) /* check if this is actually a data track: * TRUE when has data, FALSE if has no data * When NO trackDb, can't tell at this point, will check that later */ { if (tdb) { if (tdbIsContainer(tdb) || tdbIsComposite(tdb) || tdbIsCompositeView(tdb) || tdbIsSuper(tdb)) return FALSE; else return TRUE; } else return TRUE; /* might be true */ } static struct hash *dbExistsCache = NULL; /* memoize sqlDatabaseExists() below, * protectedTrack() is called once per track in a listing */ boolean protectedTrack(char *db, struct trackDb *tdb, char *tableName) /* determine if track is off-limits protected data */ { // A hub track's table name always carries the hub_<id>_ prefix, but db only // needs that same prefix reconstructed when db itself is an assembly hub // genome (accessControlInit()/trackHubDatabase() look it up by its decorated // name). When db is a real, existing database -- as it is for a hub track // that merely overlays an existing genome, e.g. a hub track on hg19 -- db // must be left alone, or accessControlInit()'s hAllocConn() aborts with // "Unknown database". if (isHubTrack(tableName)) { if (dbExistsCache == NULL) dbExistsCache = hashNew(0); struct hashEl *hel = hashLookup(dbExistsCache, db); boolean isRealDb; if (hel == NULL) { isRealDb = sqlDatabaseExists(db); hashAdd(dbExistsCache, db, isRealDb ? intToPt(1) : intToPt(0)); } else isRealDb = ptToInt(hel->val); if (! isRealDb) { char buffer[4096]; safef(buffer, sizeof buffer, "hub_%d_%s",hubIdFromTrackName(tableName), db); db = cloneString(buffer); } } return cartTrackDbIsAccessDenied(db, tableName) || cartTrackDbIsNoGenome(db, tableName); } boolean isWiggleDataTable(char *type) /* is this a wiggle data track table */ { if (startsWith("wig", type)) { if (startsWith("wigMaf", type)) return FALSE; else return TRUE; } else return FALSE; } char *chrOrAlias(char *db, char *hubUrl) /* get incoming chr name, may be an alias, return the native chr name * might be given a db, maybe not * might be given a hubUrl, maybe not */ { char *cartChr = cgiOptionalString("chrom"); if (isEmpty(cartChr)) return NULL; char *chrom = cartChr; if (isEmpty(hubUrl)) { if (isEmpty(db)) return chrom; chromAliasSetup(db); chrom = hgOfficialChromName(db, chrom); } else { /* not sure if the 'curated' hub situation has been solved yet if (sameString("hs1", db)) { chromAliasSetup("hub_25359_hs1"); } else { chromAliasSetup(db); } */ chrom = chromAliasFindNative(chrom); } if (isEmpty(chrom)) // can't find it here, return the name from the cart chrom = cartChr; return chrom; } struct trackHubGenome *hubGenome = NULL; void hubAliasSetup(struct trackHubGenome *hubGenome) /* see if this hub has an alias file and run chromAliasSetupBb() for it */ { if (hubGenome->settingsHash) { char *aliasFile = hashFindVal(hubGenome->settingsHash, "chromAliasBb"); char *absFileName = NULL; if (aliasFile) absFileName = trackHubRelativeUrl((hubGenome->trackHub)->url, aliasFile); if (absFileName) { chromAliasSetupBb(NULL, absFileName); } } } static struct dyString *textOutput = NULL; void textLineOut(char *lineOut) /* accumulate text lines for output in the dyString textOutput */ { if (NULL == textOutput) { textOutput = dyStringNew(0); boolean doContext = (cgiOptionalInt(argSkipContext, 0)==0); if (doContext) { char outString[1024]; time_t timeNow = time(NULL); struct tm tm; gmtime_r(&timeNow, &tm); safef(outString, sizeof(outString), "# downloadTime: \"%d:%02d:%02dT%02d:%02d:%02dZ\"", 1900+tm.tm_year, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); dyStringPrintf(textOutput, "%s\n", outString); safef(outString, sizeof(outString), "# downloadTimeStamp: %lld", (long long) timeNow); dyStringPrintf(textOutput, "%s\n", outString); } } dyStringPrintf(textOutput, "%s\n", lineOut); } void textFinishOutput() /* all done with text output, print it all out */ { puts("X-Content-Type-Options: nosniff"); puts("Content-Type:text/plain\n"); printf("%s", dyStringCannibalize(&textOutput)); } static char *thisHostName() /* Return this machine's own hostname via gethostname(). Unlike hHttpHost(), * which reflects the client-supplied HTTP_HOST/Host: header, this can't be * spoofed by the request and doesn't change depending on which round-robin * name (e.g. genome.ucsc.edu) the client used to reach this box -- using * hHttpHost() here made onGenomeRRMachine() misclassify RR machines reached * via the genome.ucsc.edu name, sending them into an infinite self-relay * loop. */ { static char host[256]; static boolean init = FALSE; if (!init) { if (gethostname(host, sizeof(host)) != 0) host[0] = '\0'; init = TRUE; } return host; } boolean inUcscEduDomain() /* Return TRUE if this machine is configured as belonging to the ucsc.edu * domain, per hg.conf's central.domain setting (the same setting used for * the cross-host login cookie domain). This is deployment config rather * than something derived from the machine's own OS hostname/DNS: a box's * local/cloud-assigned hostname and its UCSC-facing name (e.g. * genome-euro.ucsc.edu) can be unrelated DNS labels with no way to bridge * them via gethostname()/getaddrinfo(). */ { char *domain = cfgOption(CFG_CENTRAL_DOMAIN); return (domain != NULL && endsWith(domain, ".ucsc.edu")); } boolean onGenomeRRMachine() /* Return TRUE if running on one of the genome.ucsc.edu round-robin * machines (hgw0, hgw1, hgw2, ...), which carry SQL grants on * hgcentral.gbMembers. Only meaningful within inUcscEduDomain(). */ { if (hIsPrivateHost()) // stay on localhost for hgwdev and hgwbeta return TRUE; char *hostName = thisHostName(); if (isEmpty(hostName)) return FALSE; if (startsWith("hgwbeta", hostName)) return TRUE; if (!startsWith("hgw", hostName)) return FALSE; char afterHgw = hostName[3]; return (afterHgw >= '0' && afterHgw <= '9'); } char *submitOttoRequest(char *requestType, char *fromDb, char *toDb, char *email, char *comment) /* Record a row in the ottoRequest table via hConnectCentral(), applying the * liftOver duplicate/daily-rate guards when requestType is "liftOver", or a * plain insert when requestType is "assembly". Returns a status string: * "disabled", "duplicate", "rateLimited", "accepted", or "error". Never * apiErrAbort()s -- used both for direct local calls (this host has * hgcentral write grants) and to answer relaySubmitOttoRequest() calls. */ { char *ottoTable = cfgOption("ottoTable"); if (isEmpty(ottoTable)) return "disabled"; struct sqlConnection *conn = hConnectCentral(); if (!sqlTableExists(conn, ottoTable)) { hDisconnectCentral(&conn); return "disabled"; } char *status = "error"; if (sameString(requestType, "liftOver")) { struct dyString *dq = dyStringNew(0); sqlDyStringPrintf(dq, "SELECT COUNT(*) FROM %s WHERE requestType='liftOver' AND " "((fromDb='%s' AND toDb='%s') OR (fromDb='%s' AND toDb='%s'))", ottoTable, fromDb, toDb, toDb, fromDb); int dupCount = sqlQuickNum(conn, dyStringCannibalize(&dq)); if (dupCount > 0) status = "duplicate"; else { char *limitStr = cfgOption("liftDailyLimit"); int dailyLimit = isNotEmpty(limitStr) ? atoi(limitStr) : 0; boolean rateLimited = FALSE; if (dailyLimit > 0) { struct dyString *q = dyStringNew(0); sqlDyStringPrintf(q, "SELECT COUNT(*) FROM %s WHERE requestType='liftOver' AND email='%s' " "AND DATE(requestTime) = CURDATE()", ottoTable, email); int todayCount = sqlQuickNum(conn, dyStringCannibalize(&q)); rateLimited = (todayCount >= dailyLimit); } if (rateLimited) status = "rateLimited"; else { /* Atomic insert with duplicate re-check, closing the race window * between the SELECT above and this INSERT. */ struct dyString *update = dyStringNew(0); sqlDyStringPrintf(update, "INSERT INTO %s (requestType, fromDb, toDb, email, comment, requestTime, status, buildDir) " "SELECT 'liftOver', '%s', '%s', '%s', '%s', now(), 0, '' " "WHERE NOT EXISTS (" " SELECT 1 FROM %s WHERE requestType='liftOver' AND " " ((fromDb='%s' AND toDb='%s') OR (fromDb='%s' AND toDb='%s'))" ")", ottoTable, fromDb, toDb, email, comment, ottoTable, fromDb, toDb, toDb, fromDb); sqlUpdate(conn, dyStringCannibalize(&update)); char rowCountQuery[64]; sqlSafef(rowCountQuery, sizeof(rowCountQuery), "SELECT ROW_COUNT()"); int rowsAffected = sqlQuickNum(conn, rowCountQuery); status = (rowsAffected > 0) ? "accepted" : "duplicate"; } } } else if (sameString(requestType, "assembly")) { struct dyString *update = dyStringNew(0); sqlDyStringPrintf(update, "INSERT INTO %s (requestType, fromDb, toDb, email, comment, requestTime, status, buildDir) " "VALUES ('assembly', '%s', '%s', '%s', '%s', now(), 0, '')", ottoTable, fromDb, toDb, email, comment); sqlUpdate(conn, dyStringCannibalize(&update)); status = "accepted"; } hDisconnectCentral(&conn); return status; } char *relaySubmitOttoRequest(char *requestType, char *fromDb, char *toDb, char *email, char *comment) /* Relay an ottoRequest submission to genome.ucsc.edu's /submitOttoRequest * endpoint, for hosts that lack local hgcentral write grants (see * inUcscEduDomain()/onGenomeRRMachine()). Authenticates with the shared * hg.conf secret 'hubApi.relaySecret', which must also be configured on * genome.ucsc.edu. Returns the same status vocabulary as * submitOttoRequest(): "disabled", "duplicate", "rateLimited", "accepted", * or "error" (including when the secret isn't configured locally, or the * relay call itself fails). */ { char *secret = cfgOption("hubApi.relaySecret"); /* Guard against CR/LF in the configured secret: it goes verbatim into an * HTTP header below, and a stray newline there would inject headers. */ if (isEmpty(secret) || strpbrk(secret, "\r\n")) return "error"; struct dyString *url = dyStringNew(0); dyStringPrintf(url, "https://genome.ucsc.edu/cgi-bin/hubApi/submitOttoRequest" "?%s=%s&%s=%s&%s=%s&%s=%s&%s=%s", argRequestType, cgiEncode(requestType), argFromGenome, cgiEncode(fromDb), argToGenome, cgiEncode(toDb), argEmail, cgiEncode(email), argComment, cgiEncode(comment)); struct dyString *header = dyStringNew(0); dyStringPrintf(header, "X-Relay-Secret: %s\r\n", secret); int sd = netOpenHttpExt(dyStringContents(url), "GET", dyStringContents(header)); dyStringFree(&header); if (sd < 0) { dyStringFree(&url); return "error"; } char *redirectedUrl = NULL; if (!netSkipHttpHeaderLinesWithRedirect(sd, dyStringContents(url), &redirectedUrl)) { close(sd); dyStringFree(&url); return "error"; } dyStringFree(&url); struct dyString *body = netSlurpFile(sd); close(sd); char *status = "error"; struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) { struct jsonElement *json = jsonParse(body->string); char *statusField = jsonStringField(json, "status"); if (isNotEmpty(statusField)) status = cloneString(statusField); } errCatchEnd(errCatch); errCatchFree(&errCatch); dyStringFree(&body); return status; } void apiSubmitOttoRequest(char *words[MAX_PATH_INFO]) /* Internal server-to-server endpoint: record a row in the ottoRequest table * on behalf of a hubApi host that lacks its own hgcentral write grants (see * relaySubmitOttoRequest()). Requires the shared hg.conf secret * 'hubApi.relaySecret' to match, since this performs the actual database * write without the bot-challenge cookie check that /liftRequest and * /assemblyRequest do locally -- that check already ran on the relaying * host before it got here. Always answers 200 + a JSON "status" field; * never apiErrAbort()s for a duplicate/rateLimited outcome, so the relay * caller can distinguish it from a hard failure. */ { char *secret = cfgOption("hubApi.relaySecret"); char *givenSecret = getenv("HTTP_X_RELAY_SECRET"); if (isEmpty(secret) || isEmpty(givenSecret) || !sameString(secret, givenSecret)) apiErrAbort(err403, err403Msg, "not authorized for endpoint '/submitOttoRequest"); char *requestType = cgiOptionalString(argRequestType); char *fromDb = cgiOptionalString(argFromGenome); char *toDb = cgiOptionalString(argToGenome); char *email = cgiOptionalString(argEmail); char *comment = cgiOptionalString(argComment); if (isEmpty(requestType) || isEmpty(fromDb) || isEmpty(toDb) || isEmpty(email) || isEmpty(comment)) apiErrAbort(err400, err400Msg, "must have all arguments: %s, %s, %s, %s, %s for endpoint '/submitOttoRequest", argRequestType, argFromGenome, argToGenome, argEmail, argComment); if (!sameString(requestType, "liftOver") && !sameString(requestType, "assembly")) apiErrAbort(err400, err400Msg, "unrecognized %s '%s' for endpoint '/submitOttoRequest", argRequestType, requestType); char *status = submitOttoRequest(requestType, fromDb, toDb, email, comment); struct jsonWrite *jw = apiStartOutput(); jsonWriteString(jw, "status", status); apiFinishOutput(0, NULL, jw); }