gigablast/open-source-search-engine
1
0
Fork 0
mirror of https://github.com/gigablast/open-source-search-engine.git synced 2024-10-04 12:17:35 +03:00
Code Issues Projects Releases Wiki Activity

passing qa test again

Browse Source
This commit is contained in:
mwells 2014-07-06 22:06:33 -07:00
parent 38e64a6600
commit fed7b73b9f
2 changed files with 144 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
HttpServer.cpp
View File

@ -1610,6 +1610,30 @@ bool HttpServer::sendErrorReply ( TcpSocket *s , long error , char *errmsg ,
if ( format == FORMAT_XML ) ct = "text/xml"; if ( format == FORMAT_XML ) ct = "text/xml";
if ( format == FORMAT_JSON ) ct = "application/json"; if ( format == FORMAT_JSON ) ct = "application/json";
SafeBuf xb;
if ( format != FORMAT_XML && format != FORMAT_JSON )
xb.safePrintf("<html><b>Error = %s</b></html>",errmsg );
if ( format == FORMAT_XML ) {
xb.safePrintf("<response>\n"
"\t<statusCode>%li</statusCode>\n"
"\t<statusMsg><![CDATA[", error );
xb.cdataEncode(errmsg );
xb.safePrintf("]]></statusMsg>\n"
"</response>\n");
}
if ( format == FORMAT_JSON ) {
xb.safePrintf("{\"response\":{\n"
"\t\"statusCode\":%li,\n"
"\t\"statusMsg\":\"", error );
xb.jsonEncode(errmsg );
xb.safePrintf("\"\n"
"}\n"
"}\n");
}
sb.safePrintf( sb.safePrintf(
"HTTP/1.0 %li (%s)\r\n" "HTTP/1.0 %li (%s)\r\n"
"Content-Length: %li\r\n" "Content-Length: %li\r\n"
@ -1619,34 +1643,14 @@ bool HttpServer::sendErrorReply ( TcpSocket *s , long error , char *errmsg ,
, ,
error , error ,
errmsg , errmsg ,
(long)(gbstrlen("<html><b>Error = </b></html>")+
gbstrlen(errmsg)), xb.length(),
ct , ct ,
tt ); // ctime ( &now ) , tt ); // ctime ( &now ) ,
if ( format != FORMAT_XML && format != FORMAT_JSON ) sb.safeMemcpy ( &xb );
sb.safePrintf("<html><b>Error = %s</b></html>",errmsg );
if ( format == FORMAT_XML ) {
sb.safePrintf("<response>\n"
"\t<statusCode>%li</statusCode>\n"
"\t<statusMsg><![CDATA[", error );
sb.cdataEncode(errmsg );
sb.safePrintf("]]></statusMsg>\n"
"</response>\n");
}
if ( format == FORMAT_JSON ) {
sb.safePrintf("{\"response\":{\n"
"\t\"statusCode\":%li,\n"
"\t\"statusMsg\":\"", error );
sb.jsonEncode(errmsg );
sb.safePrintf("\"\n"
"}\n"
"}\n");
}
// . move the reply to a send buffer // . move the reply to a send buffer
// . don't make sendBuf bigger than g_conf.m_httpMaxSendBufSize // . don't make sendBuf bigger than g_conf.m_httpMaxSendBufSize

147
qa.cpp
View File

@ -4,7 +4,9 @@
static long s_failures = 0; static long s_failures = 0;
bool getUrl( char *path , void (* callback) (void *state, TcpSocket *sock) ) { bool getUrl( char *path ,
void (* callback) (void *state, TcpSocket *sock) ,
char *post = NULL ) {
SafeBuf sb; SafeBuf sb;
sb.safePrintf ( "http://%s:%li%s" sb.safePrintf ( "http://%s:%li%s"
, iptoa(g_hostdb.m_myHost->m_ip) , iptoa(g_hostdb.m_myHost->m_ip)
@ -26,7 +28,13 @@ bool getUrl( char *path , void (* callback) (void *state, TcpSocket *sock) ) {
0, // proxyport 0, // proxyport
-1, // maxtextdoclen -1, // maxtextdoclen
-1, // maxotherdoclen -1, // maxotherdoclen
NULL ) ) // useragent NULL , // useragent
"HTTP/1.0" , // protocol
true , // doPost
NULL , // cookie
NULL , // additionalHeader
NULL , // fullRequest
post ) )
return false; return false;
// error? // error?
log("qa: getUrl error: %s",mstrerror(g_errno)); log("qa: getUrl error: %s",mstrerror(g_errno));
@ -37,6 +45,8 @@ bool qatest ( ) ;
void markOut ( char *reply , char *needle ) { void markOut ( char *reply , char *needle ) {
if ( ! reply ) return;
char *s = strstr ( reply , needle ); char *s = strstr ( reply , needle );
if ( ! s ) return; if ( ! s ) return;
@ -54,7 +64,7 @@ void markOut ( char *reply , char *needle ) {
} }
// do not hash // do not hash
long qahash32 ( char *s ) { long qa_hash32 ( char *s ) {
unsigned long h = 0; unsigned long h = 0;
long k = 0; long k = 0;
for ( long i = 0 ; s[i] ; i++ ) { for ( long i = 0 ; s[i] ; i++ ) {
@ -71,12 +81,14 @@ TcpSocket *s_sock = NULL;
void qatestWrapper ( void *state , TcpSocket *sock ) { void qatestWrapper ( void *state , TcpSocket *sock ) {
log("qa: got reply(%li)=%s",sock->m_readOffset,sock->m_readBuf); log("qa: got reply(%li)=%s",sock->m_readOffset,sock->m_readBuf);
// get mime // get mime
HttpMime mime; HttpMime mime;
mime.set ( sock->m_readBuf , sock->m_readOffset , NULL ); mime.set ( sock->m_readBuf , sock->m_readOffset , NULL );
// only hash content since mime has a timestamp in it // only hash content since mime has a timestamp in it
char *content = mime.getContent(); char *content = mime.getContent();
long contentLen = mime.getContentLen(); long contentLen = mime.getContentLen();
if ( content[contentLen] ) { char *xx=NULL;*xx=0; }
char *reply = sock->m_readBuf; char *reply = sock->m_readBuf;
@ -88,9 +100,12 @@ void qatestWrapper ( void *state , TcpSocket *sock ) {
// until i figure this one out, take it out // until i figure this one out, take it out
markOut ( reply , "<docsInCollection>"); markOut ( reply , "<docsInCollection>");
// until i figure this one out, take it out
markOut ( reply , "<hits>");
// make checksum. we ignore back to back spaces so this // make checksum. we ignore back to back spaces so this
// hash works for <docsInCollection>10 vs <docsInCollection>9 // hash works for <docsInCollection>10 vs <docsInCollection>9
s_replyCRC = hash32 ( content , contentLen ); s_replyCRC = qa_hash32 ( content );
// this too is used for recording the reply into a file on disk // this too is used for recording the reply into a file on disk
s_sock = sock; s_sock = sock;
@ -310,7 +325,7 @@ bool checkSpidersDone ( ) {
return false; return false;
} }
static long s_phase = -1; //static long s_phase = -1;
void checkCRC ( long needCRC ) { void checkCRC ( long needCRC ) {
@ -330,8 +345,9 @@ void checkCRC ( long needCRC ) {
return; return;
} }
const char *emsg = "qa: bad replyCRC of %li should be %li phase=%li\n"; const char *emsg = "qa: bad replyCRC of %li should be %li "
fprintf(stderr,emsg,s_replyCRC,needCRC,s_phase-1); "\n";//"phase=%li\n";
fprintf(stderr,emsg,s_replyCRC,needCRC);//,s_phase-1);
// get response on file // get response on file
SafeBuf fb1; SafeBuf fb1;
char fn1[1024]; char fn1[1024];
@ -357,6 +373,8 @@ void checkCRC ( long needCRC ) {
//static long s_rdbId2 = 0; //static long s_rdbId2 = 0;
//static long s_rdbId3 = 0; //static long s_rdbId3 = 0;
#undef usleep
// . run a series of tests to ensure that gb is functioning properly // . run a series of tests to ensure that gb is functioning properly
// . use s_urls[] array of urls for injecting and spider seeding // . use s_urls[] array of urls for injecting and spider seeding
// . contain an archive copy of all webpages in the injectme3 file and // . contain an archive copy of all webpages in the injectme3 file and
@ -365,8 +383,12 @@ void checkCRC ( long needCRC ) {
// replay later. store up to 100,000 pages in there. // replay later. store up to 100,000 pages in there.
bool qatest ( ) { bool qatest ( ) {
if ( s_phase == -1 ) { // hack
s_phase++; //goto checkdelim;
static bool s_x1 = false;
if ( ! s_x1 ) {
s_x1 = true;
return getUrl ( "/admin/delcoll?delcoll=qatest123" , return getUrl ( "/admin/delcoll?delcoll=qatest123" ,
qatestWrapper ); qatestWrapper );
} }
@ -374,8 +396,9 @@ bool qatest ( ) {
// //
// add the 'qatest123' collection // add the 'qatest123' collection
// //
if ( s_phase == 0 ) { static bool s_x2 = false;
s_phase++; if ( ! s_x2 ) {
s_x2 = true;
if ( ! getUrl ( "/admin/addcoll?addcoll=qatest123&xml=1" , if ( ! getUrl ( "/admin/addcoll?addcoll=qatest123&xml=1" ,
qatestWrapper ) ) qatestWrapper ) )
return false; return false;
@ -384,22 +407,27 @@ bool qatest ( ) {
// //
// check addcoll reply // check addcoll reply
// //
if ( s_phase == 1 ) { static bool s_x3 = false;
s_phase++; if ( ! s_x3 ) {
s_x3 = true;
checkCRC ( 238170006 ); checkCRC ( 238170006 );
} }
// hack
//goto deliminject;
// //
// inject urls, return false if not done yet // inject urls, return false if not done yet
// //
if ( s_phase == 2 ) { static bool s_x4 = false;
if ( ! s_x4 ) {
// TODO: try delimeter based injection too // TODO: try delimeter based injection too
loadUrls(); loadUrls();
static long s_ii = 0; static long s_ii = 0;
for ( ; s_ii < s_ubuf2.length()/(long)sizeof(char *) ; ) { for ( ; s_ii < s_ubuf2.length()/(long)sizeof(char *) ; ) {
// inject using html api // inject using html api
SafeBuf sb; SafeBuf sb;
sb.safePrintf("/admin/inject?c=qatest123&deleteurl=0&" sb.safePrintf("&c=qatest123&deleteurl=0&"
"format=xml&u="); "format=xml&u=");
sb.urlEncode ( s_urlPtrs[s_ii] ); sb.urlEncode ( s_urlPtrs[s_ii] );
// the content // the content
@ -409,36 +437,43 @@ bool qatest ( ) {
sb.nullTerm(); sb.nullTerm();
// pre-inc it in case getUrl() blocks // pre-inc it in case getUrl() blocks
s_ii++; s_ii++;
getUrl ( sb.getBufStart() , qatestWrapper ); getUrl("/admin/inject",qatestWrapper,sb.getBufStart());
return false; return false;
} }
s_phase++; s_x4 = true;
} }
// +the // +the
if ( s_phase == 3 ) { static bool s_x5 = false;
s_phase++; if ( ! s_x5 ) {
usleep(500000);
s_x5 = true;
getUrl ( "/search?c=qatest123&qa=1&format=xml&q=%2Bthe", getUrl ( "/search?c=qatest123&qa=1&format=xml&q=%2Bthe",
qatestWrapper ); qatestWrapper );
return false; return false;
} }
if ( s_phase == 4 ) {s_phase++;checkCRC ( 922722309 );}
static bool s_x6 = false;
if ( ! s_x6 ) { s_x6 = true ; checkCRC ( -1452050577 ); }
// sports news // sports news
if ( s_phase == 5 ) { static bool s_x7 = false;
s_phase++; if ( ! s_x7 ) {
s_x7 = true;
getUrl ( "/search?c=qatest123&qa=1&format=xml&q=sports+news", getUrl ( "/search?c=qatest123&qa=1&format=xml&q=sports+news",
qatestWrapper ); qatestWrapper );
return false; return false;
} }
if ( s_phase == 6 ) {s_phase++;checkCRC ( -442665448 ); }
static bool s_x8 = false;
if ( ! s_x8 ) { s_x8 = true; checkCRC ( -1586622518 ); }
// //
// eject/delete the urls // eject/delete the urls
// //
static long s_ii2 = 0; static long s_ii2 = 0;
for ( ; s_phase==7&&s_ii2 < s_ubuf2.length()/(long)sizeof(char *) ; ) { for ( ; s_ii2 < s_ubuf2.length()/(long)sizeof(char *) ; ) {
// reject using html api // reject using html api
SafeBuf sb; SafeBuf sb;
sb.safePrintf( "/admin/inject?c=qatest123&deleteurl=1&" sb.safePrintf( "/admin/inject?c=qatest123&deleteurl=1&"
@ -450,19 +485,66 @@ bool qatest ( ) {
getUrl ( sb.getBufStart() , qatestWrapper ); getUrl ( sb.getBufStart() , qatestWrapper );
return false; return false;
} }
if ( s_phase == 7 ) s_phase++;
// //
// make sure no results left, +the // make sure no results left, +the
// //
if ( s_phase == 8 ) { static bool s_x9 = false;
s_phase++; if ( ! s_x9 ) {
usleep(500000);
s_x9 = true;
getUrl ( "/search?c=qatest123&qa=1&format=xml&q=%2Bthe", getUrl ( "/search?c=qatest123&qa=1&format=xml&q=%2Bthe",
qatestWrapper ); qatestWrapper );
return false; return false;
} }
// seems to have <docsInCollection>2</> // seems to have <docsInCollection>2</>
if ( s_phase == 9 ) { s_phase++ ; checkCRC ( 1515313462 ); } static bool s_y1 = false;
if ( ! s_y1 ) { s_y1 = true; checkCRC ( -1672870556 ); }
// deliminject:
//
// try delimeter based injecting
//
static bool s_y2 = false;
if ( ! s_y2 ) {
s_y2 = true;
SafeBuf sb;
// delim=+++URL:
sb.safePrintf("&c=qatest123&deleteurl=0&"
"delim=%%2B%%2B%%2BURL%%3A&format=xml&u=xyz.com&"
"hasmime=1&content=");
// use injectme3 file
SafeBuf ubuf;
ubuf.load("./injectme3");
sb.urlEncode(ubuf.getBufStart());
getUrl ( "/admin/inject",qatestWrapper,sb.getBufStart());
return false;
}
// check the reply, seems to have only a single docid in it...
static bool s_y3 = false;
if ( ! s_y3 ) { s_y3 = true; checkCRC ( -1970198487 ); }
// checkdelim:
// now query check
static bool s_y4 = false;
if ( ! s_y4 ) {
usleep(500000);
s_y4 = true;
getUrl ( "/search?c=qatest123&qa=1&format=xml&q=%2Bthe",
qatestWrapper );
return false;
}
// check search results crc
static bool s_y5 = false;
if ( ! s_y5 ) { s_y5 = true; checkCRC ( -480078278 ); }
@ -498,13 +580,16 @@ bool qatest ( ) {
//if ( ! checkRdbLists ( &s_rdbId3 ) ) return false; //if ( ! checkRdbLists ( &s_rdbId3 ) ) return false;
// delete the collection // delete the collection
if ( s_phase == 10 ) { static bool s_fee = false;
s_phase++; if ( ! s_fee ) {
s_fee = true;
return getUrl ( "/admin/delcoll?delcoll=qatest123" , return getUrl ( "/admin/delcoll?delcoll=qatest123" ,
qatestWrapper ); qatestWrapper );
} }
if ( s_phase == 11 ) { static bool s_fee2 = false;
if ( ! s_fee2 ) {
s_fee2 = true;
fprintf(stderr,"\n\n\nSUCCESSFULLY COMPLETED QA TEST\n\n\n"); fprintf(stderr,"\n\n\nSUCCESSFULLY COMPLETED QA TEST\n\n\n");
exit(0); exit(0);
} }