#include "Json.h" #include "SafeBuf.h" class JsonItem *Json::addNewItem () { JsonItem *ji = (JsonItem *)m_sb.getBuf(); if ( m_sb.m_length + (long)sizeof(JsonItem) > m_sb.m_capacity ) { log("json: preventing buffer breach"); return NULL; } // otherwise we got room m_sb.incrementLength(sizeof(JsonItem)); if ( m_prev ) m_prev->m_next = ji; ji->m_prev = m_prev; ji->m_next = NULL; // we are the new prev now m_prev = ji; // value null for now ji->m_type = JT_NULL; // parent on stack JsonItem *parent = NULL; if ( m_stackPtr > 0 ) parent = m_stack[m_stackPtr-1]; ji->m_parent = parent; // . if our parent was an array, we are an element in that array // . if it is an array of objects, then the name will be overwritten if ( parent ) { // && parent->m_type == JT_ARRAY ) { // inherit object name from parent ji->m_name = parent->m_name; ji->m_nameLen = parent->m_nameLen; } return ji; } JsonItem *Json::getFirstItem ( ) { if ( m_sb.length() <= 0 ) return NULL; return (JsonItem *)m_sb.getBufStart(); } JsonItem *Json::getItem ( char *name ) { JsonItem *ji = getFirstItem(); // traverse the json for ( ; ji ; ji = ji->m_next ) { // just get STRINGS or NUMS if ( ji->m_type != JT_STRING && ji->m_type != JT_NUMBER ) continue; // check name char *name2 = ji->m_name; if ( strcmp(name2,name) == 0 ) return ji; } return NULL; } #include "Mem.h" // gbstrlen() JsonItem *Json::parseJsonStringIntoJsonItems ( char *json , long niceness ) { m_prev = NULL; m_stackPtr = 0; m_sb.purge(); JsonItem *ji = NULL; // how much space will we need to avoid any reallocs? char *p = json; bool inQuote = false; long need = 0; for ( ; *p ; p++ ) { // ignore any escaped char. also \x1234 if ( *p == '\\' ) { if ( p[1] ) p++; continue; } if ( *p == '\"' ) inQuote = ! inQuote; if ( inQuote ) continue; if ( *p == '{' || *p == ',' || *p == '[' || *p == ':' ) // +1 for null terminating string of each item need += sizeof(JsonItem) +1; } // plus the length of the string to store it decoded etc. need += p - json; // plus a \0 for the value and a \0 for the name of each jsonitem need += 2; // prevent cores for now need += 10; // . to prevent safebuf from reallocating do this // . safeMemcpy() calls reserve(m_length+len) and reserves // tries to alloc m_length + (m_length+len) so since, // m_length+len should never be more than "need" we need to // double up here need *= 2; // this should be enough if ( ! m_sb.reserve ( need ) ) return NULL; // for testing if we realloc char *mem = m_sb.getBufStart(); long size; char *NAME = NULL; long NAMELEN = 0; // reset p p = json; // json maybe bad utf8 causing us to miss the \0 char, so use "pend" char *pend = json + gbstrlen(json); // scan for ( ; p < pend ; p += size ) { // get size size = getUtf8CharSize ( p ); // skip spaces if ( is_wspace_a (*p) ) continue; // skip commas if ( *p == ',' ) continue; // did we hit a '{'? that means the existing json item // is a parent of the item(s) inside the {}'s if ( *p == '{' ) { // if ji is non-null it must be a name like in // \"stats\":{\"fetchTime\":2069,....} // . this indicates the start of a json object // . addNewItem() will push the current item on stack ji = addNewItem(); if ( ! ji ) return NULL; // current ji is an object type then ji->m_type = JT_OBJECT; // set the name ji->m_name = NAME; ji->m_nameLen = NAMELEN; // this goes on the stack if ( m_stackPtr >= MAXJSONPARENTS ) return NULL; m_stack[m_stackPtr++] = ji; // and null this ji = NULL; continue; } // pop the stack? if ( *p == '}' ) { // just pop it and restore name cursor if ( m_stackPtr > 0 ) { JsonItem *px = m_stack[m_stackPtr-1]; NAME = px->m_name; NAMELEN = px->m_nameLen; m_stackPtr--; } continue; } // array of things? if ( *p == '[' ) { // make a newitem to put on stack ji = addNewItem(); if ( ! ji ) return NULL; // current ji is an object type then ji->m_type = JT_ARRAY; // set the name ji->m_name = NAME; ji->m_nameLen = NAMELEN; // this goes on the stack if ( m_stackPtr >= MAXJSONPARENTS ) return NULL; m_stack[m_stackPtr++] = ji; ji = NULL; continue; } // pop the stack? if ( *p == ']' ) { // just pop it and restore name cursor if ( m_stackPtr > 0 ) { JsonItem *px = m_stack[m_stackPtr-1]; NAME = px->m_name; NAMELEN = px->m_nameLen; m_stackPtr--; } continue; } // a quote? if ( *p == '\"' ) { // find end of quote char *end = p + 1; for ( ; *end ; end++ ) { // skip two chars if escaped if ( *end == '\\' && end[1] ) { end++; continue; } // this quote is unescaped then if ( *end == '\"' ) break; } // field? char *x = end + 1; // skip spaces for ( ; *x && is_wspace_a(*x) ; x++ ); // define the string char *str = p + 1; long slen = end - str; // . if a colon follows, it was a field if ( *x == ':' ) { // let's push this now so we can \0 term char *savedStr = m_sb.getBuf(); m_sb.safeMemcpy ( str , slen ); m_sb.pushChar('\0'); // just set the name cursor NAME = savedStr;//str; NAMELEN = slen; } // . otherwise, it was field value, so index it // . TODO: later make field names compounded to // better represent nesting? else { // make a new one in safebuf. our // parent will be the array type item. ji = addNewItem(); if ( ! ji ) return NULL; // we are a string ji->m_type = JT_STRING; // use name cursor ji->m_name = NAME; ji->m_nameLen = NAMELEN; // get length decoded long curr = m_sb.length(); // store decoded string right after jsonitem if ( !m_sb.safeDecodeJSONToUtf8 (str,slen, niceness )) return NULL; // store length decoded json ji->m_valueLen = m_sb.length() - curr; // end with a \0 m_sb.pushChar('\0'); // ok, this one is done ji = NULL; } // skip over the string size = 0; p = x; continue; } // true or false? if ( (*p == 't' && strncmp(p,"true",4)==0) || (*p == 'f' && strncmp(p,"false",5)==0) ) { // make a new one ji = addNewItem(); if ( ! ji ) return NULL; // copy the number as a string as well long curr = m_sb.length(); // what is the length of it? long slen = 4; ji->m_valueLong = 1; ji->m_valueDouble = 1.0; if ( *p == 'f' ) { slen = 5; ji->m_valueLong = 0; ji->m_valueDouble = 0; } // store decoded string right after jsonitem if ( !m_sb.safeDecodeJSONToUtf8 (p,slen,niceness)) return NULL; // store length decoded json ji->m_valueLen = m_sb.length() - curr; // end with a \0 m_sb.pushChar('\0'); ji->m_type = JT_NUMBER; // use name cursor ji->m_name = NAME; ji->m_nameLen = NAMELEN; ji = NULL; // skip over the string size = 1; //p = end; continue; } // if we hit a digit they might not be in quotes like // "crawled":123 if ( is_digit ( *p ) || // like .123 ? ( *p == '.' && is_digit(p[1]) ) ) { // find end of the number char *end = p + 1; // . allow '.' for decimal numbers // . TODO: allow E for exponent for ( ; *end && (is_digit(*end) || *end=='.');end++) ; // define the string char *str = p; long slen = end - str; // make a new one ji = addNewItem(); if ( ! ji ) return NULL; // decode //char c = str[slen]; //str[slen] = '\0'; ji->m_valueLong = atol(str); ji->m_valueDouble = atof(str); // copy the number as a string as well long curr = m_sb.length(); // store decoded string right after jsonitem if ( !m_sb.safeDecodeJSONToUtf8 ( str, slen,niceness)) return NULL; // store length decoded json ji->m_valueLen = m_sb.length() - curr; // end with a \0 m_sb.pushChar('\0'); //str[slen] = c; ji->m_type = JT_NUMBER; // use name cursor ji->m_name = NAME; ji->m_nameLen = NAMELEN; ji = NULL; // skip over the string size = 0; p = end; continue; } } // for testing if we realloc char *memEnd = m_sb.getBufStart(); if ( mem != memEnd ) { char *xx=NULL;*xx=0; } return (JsonItem *)m_sb.getBufStart(); } void Json::test ( ) { char *json = "{\"tags\":[\"Apple Inc.\",\"Symbian\",\"IPad\",\"Music\"],\"summary\":\"Good timing and shrewd planning have played as much of a role as innovative thinking for the Silicon Valley juggernaut.\",\"icon\":\"http://www.onlinemba.com/wp-content/themes/onlinemba/assets/img/ico/apple-touch-icon.png\",\"text\":\"How did Apple rise through the ranks to become the world’s most profitable tech company? As it turns out, good timing and shrewd planning have played as much of a role as innovative thinking for the Silicon Valley juggernaut.For example, take the first MP3 player — MPMan, produced by South Korea-based SaeHan Information Systems. MPMan appeared in 1998, three years before the first iPods were released. As the original pioneer of portable MP3 player technology, SaeHan spent a good deal of time in court negotiating terms of use with various record companies. By 2001, a clear legal precedent was set for MP3 access — allowing Apple to focus less on courtroom proceedings and more on cutting-edge marketing campaigns for their new product." "When all else fails, they buy it: While iPads had fan boys salivating in the streets –the technology has been around for decades. One of the most obvious precursors to the iPad is FingerWorks, a finger gesture operated keyboard with a mouse very similar to Apple’s iPad controller. Fingerworks was bought in 2005 by none other than Apple – not surprisingly a couple years before the release of the iPhone and later the iPad. Of course, this isn’t to say that Apple doesn’t deserve to be the most valuable tech company in the world – just that innovation isn’t always about being first or best, sometimes, it’s just perception.\",\"stats\":{\"fetchTime\":2069,\"confidence\":\"0.780\"},\"type\":\"article\",\"meta\":{\"twitter\":{\"twitter:creator\":\"@germanny\",\"twitter:domain\":\"OnlineMBA.com\",\"twitter:card\":\"summary\",\"twitter:site\":\"@OnlineMBA_com\"},\"microdata\":{\"itemprop:image\":\"http://www.onlinemba.com/wp-content/uploads/2013/02/apple-innovates-featured-150x150.png\"},\"title\":\"3 Ways Apple Actually Innovates - OnlineMBA.com\",\"article:publisher\":\"https://www.facebook.com/OnlineMBAcom\",\"fb:app_id\":\"274667389269609\",\"og\":{\"og:type\":\"article\",\"og:title\":\"3 Ways Apple Actually Innovates - OnlineMBA.com\",\"og:description\":\"Good timing and shrewd planning have played as much of a role as innovative thinking for the Silicon Valley juggernaut.\",\"og:site_name\":\"OnlineMBA.com\",\"og:image\":\"http://www.onlinemba.com/wp-content/uploads/2013/02/apple-innovates-featured-150x150.png\",\"og:locale\":\"en_US\",\"og:url\":\"http://www.onlinemba.com/blog/3-ways-apple-innovates\"}},\"human_language\":\"en\",\"url\":\"http://www.onlinemba.com/blog/3-ways-apple-innovates\",\"title\":\"3 Ways Apple Actually Innovates\",\"textAnalysis\":{\"error\":\"Timeout during text analysis\"},\"html\":\"

How did Apple rise" "\",\"supertags\":[{\"id\":856,\"positions\":[[7,12],[41,46],[663,668],[776,781],[1188,1193],[1380,1385],[1645,1650],[1841,1848],[2578,2583],[2856,2863],[2931,2936]],\"name\":\"Apple Inc.\",\"score\":0.8,\"contentMatch\":1,\"categories\":{\"1752615\":\"Home computer hardware companies\",\"27841529\":\"Technology companies of the United States\",\"33847259\":\"Publicly traded companies of the United States\",\"15168154\":\"Mobile phone manufacturers\",\"732736\":\"Retail companies of the United States\",\"9300270\":\"Apple Inc.\",\"23568549\":\"Companies based in Cupertino, " "California\",\"34056227\":\"Article Feedback 5\",\"37595560\":\"1976 establishments in California\",\"7415072\":\"Networking hardware companies\",\"699547\":\"Computer hardware companies\",\"37191508\":\"Software companies based in the San Francisco Bay Area\",\"855278\":\"Electronics companies\",\"5800057\":\"Steve Jobs\",\"7652766\":\"Display technology companies\",\"14698378\":\"Warrants issued in Hong Kong Stock Exchange\",\"4478067\":\"Portable audio player manufacturers\",\"31628257\":\"Multinational companies headquartered in the United States\",\"732825\":\"Electronics companies of the United States\",\"733759\":\"Computer companies of the United States\",\"6307421\":\"Companies established in 1976\"},\"type\":1,\"senseRank\":1,\"variety\":0.21886792452830184,\"depth\":0.6470588235294117},{\"id\":25686223,\"positions\":[[895,902],[2318,2325]],\"name\":\"Symbian\",\"score\":" "0.8,\"contentMatch\":0.9162303664921466,\"categories\":{\"33866248\":\"Nokia platforms\",\"20290726\":\"Microkernel-based operating systems\",\"39774425\":\"ARM operating systems\",\"2148723\":\"Real-time operating systems\",\"953043\":\"Smartphones\",\"10817505\":\"History of software\",\"17862682\":\"Mobile phone operating systems\",\"33569166\":\"Accenture\",\"2150815\":\"Embedded operating systems\",\"22533699\":\"Symbian OS\",\"22280474\":\"Mobile operating systems\"},\"type\":1,\"senseRank\":1,\"variety\":0.6566037735849057,\"depth\":0.6470588235294117},{\"id\":25970423,\"positions\":[[2639,2644],[2771,2775],[2864,2868]],\"name\":\"IPad\",\"score\":0.8,\"contentMatch\":1,\"categories\":{\"33578068\":\"Products introduced " "in 2010\",\"18083009\":\"Apple personal digital assistants\",\"23475157\":\"Touchscreen portable media players\",\"30107877\":\"IPad\",\"9301031\":\"Apple Inc. hardware\",\"27765345\":\"IOS (Apple)\",\"26588084\":\"Tablet computers\"},\"type\":1,\"senseRank\":1,\"variety\":0.49056603773584906,\"depth\":0.5882352941176471},{\"id\":18839,\"positions\":[[1945,1950],[2204,2209]],\"name\":\"Music\",\"score\":0.7,\"contentMatch\":1,\"categories\":{\"991222\":\"Performing arts\",\"693016\":\"Entertainment\",\"691484\":\"Music\"},\"type\":1,\"senseRank\":1,\"variety\":0.22264150943396221,\"depth\":0.7058823529411764}],\"media\":[{\"pixelHeight\":350,\"link\":\"http://www.onlinemba.com/wp-content/uploads/2013/02/apple-innovates-invert-350x350.png\",\"primary\":\"true\",\"pixelWidth\":350,\"type\":\"image\"}]}"; long niceness = 0; JsonItem *ji = parseJsonStringIntoJsonItems ( json , niceness ); // print them out? //log("json: type0=%li",(long)ji->m_type); // sanity test if ( ji->m_type != 6 ) { char *xx=NULL;*xx=0; } return; } bool JsonItem::getCompoundName ( SafeBuf &nameBuf ) { // reset, but don't free mem etc. just set m_length to 0 nameBuf.reset(); // get its full compound name like "meta.twitter.title" JsonItem *p = this;//ji; char *lastName = NULL; char *nameArray[20]; long numNames = 0; for ( ; p ; p = p->m_parent ) { // empty name? if ( ! p->m_name ) continue; if ( ! p->m_name[0] ) continue; // dup? can happen with arrays. parent of string // in object, has same name as his parent, the // name of the array. "dupname":[{"a":"b"},{"c":"d"}] if ( p->m_name == lastName ) continue; // update lastName = p->m_name; // add it up nameArray[numNames++] = p->m_name; // breach? if ( numNames < 15 ) continue; log("build: too many names in json tag"); break; } // assemble the names in reverse order which is correct order for ( long i = 1 ; i <= numNames ; i++ ) { // copy into our safebuf if ( ! nameBuf.safeStrcpy ( nameArray[numNames-i]) ) return false; // separate names with periods if ( ! nameBuf.pushChar('.') ) return false; } // remove last period nameBuf.removeLastChar('.'); // and null terminate if ( ! nameBuf.nullTerm() ) return false; // change all :'s in names to .'s since : is reserved! char *px = nameBuf.getBufStart(); for ( ; *px ; px++ ) if ( *px == ':' ) *px = '.'; return true; } // is this json item in an array of json items? bool JsonItem::isInArray ( ) { JsonItem *p = this;//ji; for ( ; p ; p = p->m_parent ) { // empty name? it's just a "value item" then, i guess. //if ( ! p->m_name ) continue; //if ( ! p->m_name[0] ) continue; if ( p->m_type == JT_ARRAY ) return true; } return false; } // convert nubers and bools to strings for this one char *JsonItem::getValueAsString ( long *valueLen ) { // strings are the same if ( m_type == JT_STRING ) { *valueLen = getValueLen(); return getValue(); } // numbers... static char s_numBuf[64]; if ( (float)m_valueLong == m_valueDouble ) { *valueLen = sprintf ( s_numBuf,"%li", m_valueLong ); return s_numBuf; } *valueLen = sprintf ( s_numBuf,"%f", m_valueDouble ); return s_numBuf; }