mirror of
https://github.com/gigablast/open-source-search-engine.git
synced 2024-10-04 20:27:43 +03:00
c911a606c9
matches2.h and matches2.cpp to avoid potential confusion with Matches.h and Matches.cpp files.
1652 lines
64 KiB
C++
1652 lines
64 KiB
C++
// http://www.propeciauk.co.uk/links.htm
|
||
// http://www.hendersonvillehomepro.com/FavoriteLinks/Default.aspx
|
||
// http://www.viacreme-viacream-viagra.com/health/pharmacies.htm
|
||
// are the same description for viagrapunch.com. why did they not cancel?
|
||
|
||
#include "linkspam.h"
|
||
#include "Url.h"
|
||
#include "Linkdb.h"
|
||
//#include "TitleRec.h"
|
||
#include "Unicode.h"
|
||
#include "matches2.h"
|
||
#include "Categories.h"
|
||
|
||
bool isLinkChain ( Xml *xml , Url *linker , Url *linkee , long linkNode ,
|
||
char **note ) ;
|
||
|
||
// . here's some additional things to mark it as a log page, but these
|
||
// depend on the content of the page, not the url itself.
|
||
// . fields: string, stringLen, id, section?
|
||
// . section is "1" if the substring identifies the start of a comment
|
||
// section, so that any links above that identifier should be
|
||
// consider good, and any below, should be considered bad links.
|
||
// Otherwise, if section is 0, if the match occurs anywhere on the
|
||
// page then all links on the page should be considered bad.
|
||
static Needle s_needles1[] = {
|
||
{"open.thumbshots.org" , 0 , 0 , 0 , 0 , NULL , 0 , NULL } ,
|
||
//{"google-ad" , 0 , 0 , 0 , 0 , NULL } ,
|
||
// indicates search results page
|
||
// this often directly precedes the comment section
|
||
{"[trackback" , 0 , 1 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"comtext" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"comment" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"coment" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"trackback" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"ping" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"followup" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"response" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
// this can signify a blog entry, not just a comment
|
||
//{"class=\"entry" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
// these seem to be more indicative of posted comments
|
||
{"class=\"posted" , 0 , 8 , 1 , 0 , NULL , 0 },
|
||
{"id=\"posted" , 0 , 8 , 1 , 0 , NULL , 0 },
|
||
{"name=\"posted" , 0 , 8 , 1 , 0 , NULL , 0 },
|
||
// annoying little textbox thingy
|
||
{"class=\"shoutbox" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"id=\"comment" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"id=\"coment" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"id=\"trackback" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"id=\"ping" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"id=\"followup" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"id=\"response" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"name=\"comment" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"name=\"coment" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"name=\"trackback" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"name=\"ping" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"name=\"followup" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"name=\"response" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
// a lot of the comment boards can be identified because
|
||
// they have a bunch of mailto links, one before each comment
|
||
//{"href=\"mailto" , 0 , 8 , 1 , 0 , NULL , 0 },
|
||
//{"href=mailto" , 0 , 8 , 1 , 0 , NULL , 0 },
|
||
// wikipedias
|
||
{"div class=\"editsection" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"action=edit" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
// message boards
|
||
{"anonymous user" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"anonymer user" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"date posted" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"post your notice" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"edit this page" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
// edit</a><br>
|
||
{"edit<a]br" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
// link to edit a comment
|
||
{">edit</a" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
// these often indicate blog entries, not just comments
|
||
//{"postedon" , 0 , 10, 1 , 0 , NULL , 0 },
|
||
//{"posted by " , 0 , 10, 1 , 0 , NULL , 0 },
|
||
//{"posted at " , 0 , 10, 1 , 0 , NULL , 0 },
|
||
{"reply with quote" , 0 , 9 , 0 , 0 , NULL , 0 , NULL } ,
|
||
{">post a reply" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"post reply" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit post" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">post message" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">post a comment" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">leave a comment" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">post comments" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
// Comments</font> (0) after each posted entry...
|
||
//{">comments<" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"comments: <" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"comments:<" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
//{"comment:" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"reacties:" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"comentarios:" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"coment<EFBFBD>rios:" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{">message:" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">mensagem:" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">faca seu comentario" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">fa<66>a seu coment<6E>rio" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
// comment add in german
|
||
{">Kommentar hinzuf" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"rate this link" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"link submit" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"links directory" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">add my comment" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
// title of the text area box
|
||
{">your comment" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"your comment<" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">comment by" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{">scrivi un commento" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">scrivi il tuo commento" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add comment" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"trackbacks for the art" , 0 , 12, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"these trackbacks have been re", 0 , 13, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"trackback pings" , 0 , 13, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"read the rest of this com" , 0 , 13, 1 , 0 , NULL , 0 , NULL } ,
|
||
// that was the opinion of ...
|
||
{"das war die meinung von" , 0 , 13, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"resource partner" , 0 , 49, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"partner link" , 0 , 50, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"partner site" , 0 , 51, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"sign the guestbook" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
//{"add new comment" , 0 , 14, 0 , 0 , NULL , 0 },
|
||
//{"add message" , 0 , 14, 0 , 0 , NULL , 0 },
|
||
// tagboard software allows free submits. it has this in
|
||
// an html comment tag...
|
||
{"2002 natali ardianto" , 0 , 14, 0 , 0 , NULL , 0 , NULL } ,
|
||
// guestbooks
|
||
{"guestbook</title" , 0 , 13, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"gastenboek</title" , 0 , 13, 0 , 0 , NULL , 0 , NULL } ,
|
||
// link management software puts a search box on there
|
||
{"search our links" , 0 , 14, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"find all words option" , 0 , 14, 0 , 0 , NULL , 0 , NULL } ,
|
||
// link exchange indicators
|
||
{"link you want to share" , 0 , 14, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"link trader" , 0 , 14, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"link exchange" , 0 , 15, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"link partner" , 0 , 16, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"link xchange" , 0 , 17, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"link swap" , 0 , 18, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"links trader" , 0 , 19, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"links exchange" , 0 , 20, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"links partner" , 0 , 21, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"links xchange" , 0 , 22, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"links swap" , 0 , 23, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"list your site" , 0 , 26, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add your web site" , 0 , 24, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add your website" , 0 , 25, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add your site" , 0 , 26, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add your link" , 0 , 27, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add your url" , 0 , 28, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add site" , 0 , 28, 0 , 0 , NULL , 0 , NULL } ,
|
||
// email the webmaster to have your link on this page
|
||
{"have your link" , 0 , 28, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add a web site" , 0 , 29, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add a website" , 0 , 30, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add a site" , 0 , 31, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add a link" , 0 , 32, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add a url" , 0 , 33, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"adding your web site" , 0 , 34, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"adding your website" , 0 , 35, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"adding your site" , 0 , 36, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"adding your link" , 0 , 37, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"adding your url" , 0 , 38, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"adding a web site" , 0 , 39, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"adding a website" , 0 , 40, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"adding a site" , 0 , 41, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"adding a link" , 0 , 42, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"adding a url" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add url" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add resource" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add link" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add free link" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"addlink" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"suggest a site" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"swap links" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"considered for addition" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"we are not affiliated" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"have a site to add" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit your web site" , 0 , 34, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit your website" , 0 , 35, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit your site" , 0 , 36, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit your link" , 0 , 37, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit your url" , 0 , 38, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit a web site" , 0 , 39, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit a website" , 0 , 40, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit a site" , 0 , 41, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit a link" , 0 , 42, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit link" , 0 , 42, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit a url" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
// . article spammers using article-emporium.com, etc.
|
||
// . these articles get circulated into regular websites
|
||
{"submit your article" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit articles" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"submit an article" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"for any feedback contact" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"for any feedback mail" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"for any feedback email" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"other articles that might" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"is a freelance" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"author is an amateur" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"article source" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"word count:" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"for additional information on", 0 , 43, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"for more information on" , 0 , 43, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"for further assistance visit" , 0 , 43, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"article submitted on" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"please rate this" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"rate the article" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
//{"how would you rate" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add rating" , 0 , 43, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"trade text link" , 0 , 44, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"trade link" , 0 , 45, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"exchange link" , 0 , 46, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"exchanging link" , 0 , 47, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"reciprocal link" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
|
||
// new stuff
|
||
{">sponsors<" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">sponsor<" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">sponsored<" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">submit site<" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{": sponsor" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"/sponsor/" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"*sponsors*" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">payperpost" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"sponsored post" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"sponsored flag" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"sponsoredflag" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"sponsored listing" , 0 , 48, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"sponsored link" , 0 , 48, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"post is sponsor" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"paid post" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"powered by" , 0 , 48, 0 , 0 , NULL , 0 , NULL } , // wordpress
|
||
{"suggest your website" , 0 , 48, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"advertisement:" , 0 , 48, 1 , 0 , NULL , 0 , NULL }
|
||
};
|
||
|
||
// now check outlinks on the page for these substrings
|
||
static Needle s_needles2[] = {
|
||
{"cyber-robotics.com" , 0 , 0 , 0 , 0 , NULL , 0 , NULL } ,
|
||
{"cyberspacehq.com" , 0 , 0 , 0 , 0 , NULL , 0 , NULL } ,
|
||
{"links4trade.com" , 0 , 0 , 0 , 0 , NULL , 0 , NULL } ,
|
||
{"searchfeed.com" , 0 , 0 , 0 , 0 , NULL , 0 , NULL } ,
|
||
{"marketnex.com" , 0 , 0 , 0 , 0 , NULL , 0 , NULL } ,
|
||
{"partnersignup" , 0 , 0 , 0 , 0 , NULL , 0 , NULL } ,
|
||
{"publisher-network" , 0 , 0 , 0 , 0 , NULL , 0 , NULL } ,
|
||
//{"amazon.com" , 0 , 0 , 0 , 0 , NULL , 0 , NULL } ,
|
||
//{"dmoz.org" , 0 , 0 , 0 , 0 , NULL , 0 , NULL } ,
|
||
//{"dmoz.com" , 0 , 0 , 0 , 0 , NULL , 0 , NULL } ,
|
||
{"linksmanager" , 0 , 0 , 0 , 0 , NULL , 0 , NULL } ,
|
||
{"changinglinks" , 0 , 0 , 0 , 0 , NULL , 0 , NULL }
|
||
};
|
||
|
||
|
||
// . we set the bit in linkdb for a doc if this returns true
|
||
// . it precludes a doc from voting if its bits is set in linkdb
|
||
// . this saves resources
|
||
// . the isLinkSpam() function is used when we have the linkee url
|
||
// . note is only set if the whole doc can not vote for some reason
|
||
// . otherwise, each outlink in "links" is assigned a "note" to indicate if
|
||
// the outlink is a spam link or not
|
||
// . returns true on success, false on error
|
||
bool setLinkSpam ( long ip ,
|
||
long *indCatIds ,
|
||
long numIndCatIds ,
|
||
Url *linker ,
|
||
long siteNumInlinks ,
|
||
Xml *xml ,
|
||
Links *links ,
|
||
bool isContentTruncated ,
|
||
long niceness ) {
|
||
// get our url
|
||
//Url *linker = tr->getUrl();
|
||
// it is critical to get inlinks from all pingserver xml
|
||
// pages regardless if they are often large pages. we
|
||
// have to manually hard-code the ping servers in for now.
|
||
if ( linker->isPingServer() ) return false;
|
||
// if the doc got truncated we may be missing valuable identifiers
|
||
// that identify the doc as a guestbook or something
|
||
if ( isContentTruncated )
|
||
return links->setAllSpamBits("doc too big");
|
||
// get linker quality
|
||
//long q = tr->getDocQuality();
|
||
// do not allow .info or .biz to vote ever for now
|
||
char *tld = linker->getTLD();
|
||
long tldLen = linker->getTLDLen();
|
||
if ( tldLen == 4 && strncmp ( tld, "info" , tldLen) == 0 && //q < 55 )
|
||
siteNumInlinks < 20 )
|
||
return links->setAllSpamBits("low quality .info linker");
|
||
if ( tldLen == 3 && strncmp ( tld, "biz" , tldLen) == 0 && //q < 55 )
|
||
siteNumInlinks < 20 )
|
||
return links->setAllSpamBits("low quality .biz linker");
|
||
|
||
// if has an outlink to dmoz-identified porn, all outlinks are spam
|
||
long *ids = indCatIds;//NULL
|
||
long nids = numIndCatIds;//0;
|
||
//if ( tr ) ids = tr->getIndCatids();
|
||
//if ( tr ) nids = tr->getNumIndCatids();
|
||
for ( long j = 0 ; j < nids ; j++ )
|
||
if ( g_categories && g_categories->isIdAdult ( ids[j] ) )
|
||
return links->setAllSpamBits("dmoz porn");
|
||
|
||
QUICKPOLL( niceness );
|
||
// do we contain a dmoz subpath in our url? that would indicate that
|
||
// we are probably a dmoz mirror!
|
||
char *zstart = linker->getPath();
|
||
long zlen = linker->getPathLen();
|
||
char *zend = zstart + zlen;
|
||
// start at the end of the path
|
||
char *z = zend-1;
|
||
// back up to previous /
|
||
for ( ; z > zstart && *z != '/' ; z-- );
|
||
// make that the new end
|
||
zend = z + 1;
|
||
// need at least 2 path components before checking... keep count
|
||
long zcount = 2;
|
||
// begin the loop
|
||
while ( z > zstart ) {
|
||
// . backup until we hit the previous /
|
||
for ( z-- ; z > zstart && *z != '/' ; z-- );
|
||
// debug
|
||
//char tmp[2000];
|
||
//memcpy(tmp,z,zend-z);
|
||
//tmp[zend-z]=0;
|
||
//log("build: path=%s",tmp);
|
||
// look it up
|
||
// "/Arts/" --> 1
|
||
if ( --zcount > 0 ) continue;
|
||
if ( g_categories->getIndexFromPath (z, zend-z) < 0 ) continue;
|
||
// consider ourselves a dmoz mirror and discount all outlinks
|
||
return links->setAllSpamBits("dmoz subpath in url");
|
||
}
|
||
|
||
QUICKPOLL( niceness );
|
||
// guestbook in hostname - domain?
|
||
char *hd = linker->getHost();
|
||
char *hd2 = linker->getDomain();
|
||
long hdlen = hd2 - hd;
|
||
if ( hd && hd2 && hdlen < 30 ) {
|
||
char c = hd[hdlen];
|
||
hd[hdlen] = '\0';
|
||
bool hasIt = false;
|
||
if ( strstr ( hd , "guestbook" ) ) hasIt = true;
|
||
hd[hdlen] = c;
|
||
if ( hasIt )
|
||
return links->setAllSpamBits("guestbook in hostname");
|
||
}
|
||
|
||
// do not allow any cgi url to vote
|
||
if ( linker->isCgi() )
|
||
return links->setAllSpamBits("path is cgi");
|
||
|
||
long plen = linker->getPathLen();
|
||
// if the page has just one rel=nofollow tag then we know they
|
||
// are not a guestbook
|
||
//if ( links->hasRelNoFollow() ) plen = 0;
|
||
if ( plen > 1 ) {
|
||
char *p = linker->getPath();
|
||
//char c = p[plen-1];
|
||
//p[plen-1] = '\0';
|
||
//bool val = false;
|
||
char *note = NULL;
|
||
if ( strncasestr ( p , "guest",plen,5) )
|
||
note = "path has guest" ;
|
||
else if ( strncasestr ( p , "cgi",plen,3) )
|
||
note = "path has cgi" ;
|
||
else if ( strncasestr ( p , "gast",plen,4) )
|
||
note = "path has gast" ;
|
||
// german
|
||
else if ( strncasestr ( p , "gaest",plen,5) )
|
||
note = "path has gaest" ;
|
||
else if ( strncasestr ( p , "gbook",plen,5) )
|
||
note = "path has gbook" ;
|
||
// vietnamese?
|
||
else if ( strncasestr ( p , "akobook",plen,7) )
|
||
note = "path has akobook" ;
|
||
else if ( strncasestr ( p , "/gb",plen,3) )
|
||
note = "path has /gb" ;
|
||
else if ( strncasestr ( p , "msg",plen,3 ) )
|
||
note = "path has msg" ;
|
||
else if ( strncasestr ( p , "messag",plen,6) )
|
||
note = "path has messag" ;
|
||
else if ( strncasestr ( p , "board",plen,5) )
|
||
note = "path has board" ;
|
||
else if ( strncasestr ( p , "coment",plen,6) )
|
||
note = "path has coment" ;
|
||
else if ( strncasestr ( p , "comment",plen,7) )
|
||
note = "path has comment" ;
|
||
else if ( strncasestr ( p , "linktrader",plen,10) )
|
||
note = "path has linktrader" ;
|
||
else if ( strncasestr ( p , "tradelinks",plen,10) )
|
||
note = "path has tradelinks" ;
|
||
else if ( strncasestr ( p , "trade-links",plen,11) )
|
||
note = "path has trade-links" ;
|
||
else if ( strncasestr ( p , "linkexchange",plen,12) )
|
||
note = "path has linkexchange" ;
|
||
else if ( strncasestr ( p , "link-exchange",plen,13 ) )
|
||
note = "path has link-exchange" ;
|
||
else if ( strncasestr ( p , "reciprocal-link",plen,15) )
|
||
note = "path has reciprocal-link";
|
||
else if ( strncasestr ( p , "reciprocallink",plen, 14) )
|
||
note = "path has reciprocallink" ;
|
||
else if ( strncasestr ( p , "/trackbacks/",plen,12 ) )
|
||
note = "path has /trackbacks/" ;
|
||
//if ( gb_strcasestr ( p , "archive" ) ) val = true;
|
||
//if ( gb_strcasestr ( p , ".asp" ) ) val = true;
|
||
//if ( gb_strcasestr ( p , ".aspx" ) ) val = true;
|
||
// these are mostly link exchange pages. no they are not!
|
||
//if ( gb_strcasestr ( p , "link" ) ) val = true;
|
||
//p[plen-1] = c;
|
||
//if ( val ) { note = "cgi or guestbook url"; return true; }
|
||
if ( note ) return links->setAllSpamBits(note);
|
||
}
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
// does title contain "web statistics for"?
|
||
long tlen ;
|
||
char *title = xml->getString ( "title" , &tlen );
|
||
if ( title && tlen > 0 ) {
|
||
// normalize title into buffer, remove non alnum chars
|
||
char buf[256];
|
||
char *d = buf;
|
||
char *dend = buf + 250;
|
||
char *s = title;
|
||
char *send = title + tlen;
|
||
while ( d < dend && s < send ) {
|
||
// remove punct
|
||
if ( ! is_alnum_a(*s) ) { s++; continue; }
|
||
*d = to_lower_a ( *s );
|
||
d++;
|
||
s++;
|
||
}
|
||
*d = '\0';
|
||
// see if it matches some catch phrases
|
||
bool val = false;
|
||
if ( strstr (buf,"webstatisticsfor" )) val = true;
|
||
if ( strstr (buf,"webserverstatisticsfor")) val = true;
|
||
else if ( strstr (buf,"usagestatisticsfor" )) val = true;
|
||
else if ( strstr (buf,"siteusageby" )) val = true;
|
||
else if ( strstr (buf,"surfstatsloganal" )) val = true;
|
||
else if ( strstr (buf,"webstarterhelpstats" )) val = true;
|
||
else if ( strstr (buf,"sitestatistics" )) val = true;
|
||
if ( val ) return links->setAllSpamBits("stats page");
|
||
}
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
/////////////////////////////////////////////////////
|
||
//
|
||
// check content for certain keywords and phrases
|
||
//
|
||
/////////////////////////////////////////////////////
|
||
|
||
//char *haystack = tr->getContent();
|
||
//long haystackSize = tr->getContentLen();
|
||
char *haystack = xml->getContent();
|
||
long haystackSize = xml->getContentLen();
|
||
|
||
// get our page quality, it serves as a threshold for some algos
|
||
//char quality = tr->getNewQuality();
|
||
|
||
//char *linkPos = NULL;
|
||
//if ( linkNode >= 0 ) linkPos = xml->getNode ( linkNode );
|
||
|
||
//if ( strstr ( linker->getUrl() , "usa_apartments1.htm") ) {
|
||
// log("hey");
|
||
// sleep(7);
|
||
//}
|
||
|
||
// loop:
|
||
// do not call them "bad links" if our link occurs before any
|
||
// comment section. our link's position therefore needs to be known,
|
||
// that is why we pass in linkPos.
|
||
// "n" is the number it matches.
|
||
long numNeedles1 = sizeof(s_needles1)/sizeof(Needle);
|
||
bool hadPreMatch;
|
||
getMatches2 ( s_needles1 ,
|
||
numNeedles1 ,
|
||
haystack ,
|
||
haystackSize ,
|
||
NULL , // linkPos ,
|
||
NULL , // &n ,
|
||
false , // stopAtFirstMatch
|
||
&hadPreMatch ,
|
||
true , // save quicktables
|
||
niceness );
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
// see if we got a hit
|
||
char *minPtr = NULL;
|
||
char *note = NULL;
|
||
for ( long i = 0 ; i < numNeedles1 ; i++ ) {
|
||
// open.thumbshots.org needs multiple counts
|
||
if ( i == 0 && s_needles1[i].m_count < 5 ) continue;
|
||
// skip if no matches on this string
|
||
if ( s_needles1[i].m_count <= 0 ) continue;
|
||
// ok, if it had its section bit set to 0 that means the
|
||
// whole page is link spam!
|
||
if ( s_needles1[i].m_isSection == 0 )
|
||
return links->setAllSpamBits(s_needles1[i].m_string );
|
||
// get the char ptr
|
||
char *ptr = s_needles1[i].m_firstMatch;
|
||
// set to the min
|
||
if ( ! minPtr || ptr < minPtr ) {
|
||
note = s_needles1[i].m_string;
|
||
minPtr = ptr;
|
||
}
|
||
}
|
||
QUICKPOLL( niceness );
|
||
|
||
// convert the char ptr into a link node following it
|
||
long aa = 0;
|
||
if ( minPtr ) aa = links->getNumLinks();
|
||
long mini = -1;
|
||
for ( long i = 0 ; i < aa ; i++ ) {
|
||
// get the link's char ptr into the content
|
||
long linkNode = links->getNodeNum(i);
|
||
char *linkPos = NULL;
|
||
if ( linkNode >= 0 ) linkPos = xml->getNode ( linkNode );
|
||
// now we can compare, if BEFORE this comment section
|
||
// indicating tag, we are NOT link spam, so continue
|
||
if ( linkPos < minPtr ) continue;
|
||
// otherwise, we are the first, stop.
|
||
mini = i;
|
||
break;
|
||
}
|
||
QUICKPOLL( niceness );
|
||
|
||
// now count all the links BELOW this match as link spam
|
||
// but everyone else is ok!
|
||
if ( minPtr && mini >= 0 )
|
||
links->setSpamBits ( note , mini );
|
||
|
||
// now check outlinks on the page for these substrings
|
||
haystack = links->getLinkBuf();
|
||
haystackSize = links->getLinkBufLen();
|
||
long numNeedles2 = sizeof(s_needles2)/sizeof(Needle);
|
||
getMatches2 ( s_needles2 ,
|
||
numNeedles2 ,
|
||
haystack ,
|
||
haystackSize ,
|
||
NULL , // linkPos,
|
||
NULL , // &n ,
|
||
false , // stopAtFirstMatch?
|
||
NULL ,
|
||
true , // save quicktables
|
||
niceness );
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
// see if we got a hit
|
||
for ( long i = 0 ; i < numNeedles2 ; i++ ) {
|
||
// skip if did not match
|
||
if ( s_needles2[i].m_count <= 0 ) continue;
|
||
// the whole doc is considered link spam
|
||
return links->setAllSpamBits(s_needles2[i].m_string);
|
||
}
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
//skiplinks:
|
||
// check for certain post tag, indicative of a comment-friendly blog
|
||
// <form method=post ... action=*comments*cgi-bin>
|
||
// <form method="post"
|
||
// action="http://www.mydomain.com/cgi-bin/mt-comments.cgi"
|
||
// name="comments_form" ...>
|
||
// <form method=POST
|
||
// action="http://peaceaction.org/wboard/wwwboard.cgi">
|
||
long nn = xml->getNumNodes();
|
||
bool gotTextArea = false;
|
||
bool gotSubmit = false;
|
||
for ( long i=0; i < nn ; i++ ) {
|
||
// <textarea> tags are bad... but only if we have not
|
||
// matched "track" or whatever from above... check for that
|
||
// if you uncommment this... otherwise you disable all blogs!
|
||
// Only do this check if we did match a comment related phrase
|
||
// in s_needles1[] BUT it was BEFORE our outlink. That
|
||
// basically means that we do *not* recognize the format of
|
||
// the comment page and so therefore need to be more
|
||
// restrictive about allowing this page to vote.
|
||
if ( ! hadPreMatch ) {
|
||
// is it a <textarea> tag?
|
||
if ( xml->getNodeId ( i ) == TAG_TEXTAREA )
|
||
gotTextArea = true;
|
||
// is it an <input> tag?
|
||
long len = 0;
|
||
if ( xml->getNodeId ( i ) == TAG_INPUT &&
|
||
xml->getString(i,"submit",&len)) gotSubmit = true;
|
||
}
|
||
// check for script tag
|
||
/*
|
||
if ( xml->getNodeId(i) == TAG_SCRIPT && quality < 80 ) {
|
||
// <script src=blah.com/fileparse.js"
|
||
// type="text/javascript"> is used to hide google
|
||
// ads, so don't allow those pages to vote either
|
||
long slen; xml->getString(i,"src",&slen);
|
||
if ( slen > 0 ) { *note = "script src"; return true; }
|
||
}
|
||
*/
|
||
if ( xml->getNodeId ( i ) != TAG_FORM ) continue;
|
||
|
||
// get the method field of this base tag
|
||
long slen;
|
||
char *s = (char *) xml->getString(i,"method",&slen);
|
||
// if not thee, skip it
|
||
if ( ! s || slen <= 0 ) continue;
|
||
//if ( slen != 4 ) continue;
|
||
// if not a post, skip it
|
||
//if ( strncasecmp ( s , "post" , 4 ) ) continue;
|
||
// get the action url
|
||
s = (char *) xml->getString(i,"action",&slen);
|
||
if ( ! s || slen <= 0 ) continue;
|
||
char c = s[slen];
|
||
s[slen]='\0';
|
||
bool val = false;
|
||
// this is a bit too strong, but i'ev seen an action of
|
||
// "cgi-bin/mt-leaveone.cgi" so we can't rely on "mt-comment"
|
||
if ( strstr ( s , "comment" ) ) val = true;
|
||
else if ( strstr ( s , "/MT/" ) ) val = true;
|
||
else if ( strstr ( s , "/mt/" ) ) val = true;
|
||
// they can have these search boxes though
|
||
if ( val && strstr ( s , "/mt/mt-search" ) ) val = false;
|
||
//else if ( strstr ( s , "cgi" ) ) val = true;
|
||
// eliminate some false positives
|
||
//if ( val && strstr ( s , "search" ) ) val = false;
|
||
s[slen] = c;
|
||
if ( val ) return links->setAllSpamBits("post page");
|
||
QUICKPOLL( niceness );
|
||
}
|
||
|
||
if ( gotTextArea && gotSubmit )
|
||
return links->setAllSpamBits("textarea tag");
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
// edu, gov, etc. can have link chains
|
||
if ( tldLen >= 3 && strncmp ( tld, "edu" , 3) == 0 ) return true;
|
||
if ( tldLen >= 3 && strncmp ( tld, "gov" , 3) == 0 ) return true;
|
||
// . allow sites in dmoz to have link chains, too
|
||
// . no, there are too many spam porn sites in dmoz
|
||
//if ( tr->getNumIndCatids() || tr->getNumCatids() ) return false;
|
||
|
||
// if linker is naughty, he cannot vote... how did he make it in?
|
||
if ( linker->isSpam() )
|
||
return links->setAllSpamBits("linker is sporny");
|
||
|
||
// . if they link to any adult site, consider them link spam
|
||
// . just consider a 100 link radius around linkNode
|
||
long nl = links->getNumLinks();
|
||
for ( long i = 0 ; i < nl ; i++ ) {
|
||
// skip if this link is internal, we will add it to linkdb
|
||
// anyway... this will save us some processing time
|
||
if ( links->isInternalDom(i) ) continue;
|
||
// otherwise, normalize it...
|
||
Url uu; uu.set ( links->getLink(i), links->getLinkLen(i) );
|
||
char *h = uu.getUrl();
|
||
// include the trailing /
|
||
char *hend = uu.getHost() + uu.getHostLen() + 1;
|
||
unsigned long hhash = hash32 ( h , hend - h );
|
||
if ( g_categories && g_categories->isInBadCat ( hhash ) ) {
|
||
links->setAllSpamBits("links to dmoz filth");
|
||
log(LOG_DEBUG,"build: %s is filthy.",uu.getUrl());
|
||
return true;
|
||
}
|
||
// take a break
|
||
QUICKPOLL ( niceness );
|
||
// . is it near sporny links? (naughty domains or lotsa -'s)
|
||
// . if we are in a list of ads, chances are good the true
|
||
// nature of the ads will emerge...
|
||
if ( uu.isSpam() ) {
|
||
links->setAllSpamBits("has sporny outlinks");
|
||
log(LOG_DEBUG,"build: %s has sporny outlinks.",
|
||
uu.getUrl());
|
||
return true;
|
||
}
|
||
|
||
// check if this url is a link chain
|
||
//if ( q >= 60 ) continue;
|
||
if ( siteNumInlinks >= 50 ) continue;
|
||
char *np = NULL;
|
||
//if ( strncmp("http://www.reliant.com",uu.getUrl(),18)==0 )
|
||
// log("hey");
|
||
// get the xml node of link #i
|
||
long xmlNode = links->getNodeNum ( i );
|
||
if ( isLinkChain ( xml , linker, &uu, xmlNode, &np ))
|
||
links->setSpamBit ( np , i );
|
||
// take a break
|
||
QUICKPOLL ( niceness );
|
||
}
|
||
return true;
|
||
}
|
||
|
||
|
||
|
||
bool isLinkSpam ( Url *linker,
|
||
long ip ,
|
||
long *indCatIds ,
|
||
long numIndCatIds ,
|
||
long siteNumInlinks ,
|
||
//TitleRec *tr,
|
||
Xml *xml,
|
||
Links *links ,
|
||
long maxDocLen ,
|
||
char **note ,
|
||
Url *linkee ,
|
||
// node position of the linkee in the linker's content
|
||
long linkNode ,
|
||
char *coll ,
|
||
long niceness ) {
|
||
// it is critical to get inlinks from all pingserver xml
|
||
// pages regardless if they are often large pages. we
|
||
// have to manually hard-code the ping servers in for now.
|
||
if ( linker->isPingServer() ) return false;
|
||
// same host linkers can be link spam (TODO: make same ip block)
|
||
// because we only allow up to 10 to vote as a single voter
|
||
if ( linkee ) {
|
||
char *h1 = linkee->getHost();
|
||
long h1len = linkee->getHostLen();
|
||
char *h2 = NULL;
|
||
long h2len = 0;
|
||
//if ( tr ) h2 = tr->getUrl()->getHost();
|
||
//if ( tr ) h2len = tr->getUrl()->getHostLen();
|
||
h2 = linker->getHost();
|
||
h2len = linker->getHostLen();
|
||
if ( h1len == h2len && strncmp ( h1 , h2 , h1len ) == 0 )
|
||
return false;
|
||
}
|
||
// do not allow .info or .biz to vote ever for now
|
||
char *tld = linker->getTLD();
|
||
long tldLen = linker->getTLDLen();
|
||
if ( tldLen == 4 && strncmp ( tld, "info" , tldLen) == 0 ) {
|
||
*note = ".info tld";
|
||
return true;
|
||
}
|
||
if ( tldLen == 3 && strncmp ( tld, "biz" , tldLen) == 0 ) {
|
||
*note = ".biz tld";
|
||
return true;
|
||
}
|
||
//long ip = tr->getIp();
|
||
char *ips = (char *)&ip;
|
||
if ( ips[0]==70 && ips[1]>=80 && ips[1]<=89 ) {
|
||
*note = "70.8*. ip";
|
||
return true;
|
||
}
|
||
|
||
// if this page fails zak's page quality algo, do not let it vote
|
||
// like if from .biz or lots of hyphens in the url or an ip-based
|
||
// url.
|
||
// now we do this on a site by site basis, more accurate that way.
|
||
// --z
|
||
//unsigned char spamScore = tr->getSpamScore();
|
||
// unsigned char spamScore ;
|
||
// spamScore = getNegativeQualityWeight ( linker ,
|
||
// xml ,
|
||
// links ,
|
||
// NULL , // words
|
||
// coll ,
|
||
// NULL , // sr ,
|
||
// NULL ,// safebuf
|
||
// 0 /*niceness*/);
|
||
// if ( spamScore >= 30 ) {
|
||
// *note = "had big spam score";
|
||
// return true;
|
||
// }
|
||
|
||
// big pages may have keywords identifying them as log pages cutoff
|
||
// so assume the worst
|
||
//if ( tr->getContentLen() > 100*1024 ) return true;
|
||
// i saw a german doc get its textarea cut out because of this, so
|
||
// we need this here
|
||
//if ( tr && tr->getContentLen() > maxDocLen ) {
|
||
if ( xml && xml->getContentLen() > maxDocLen ) {
|
||
*note ="doc too big";
|
||
return true;
|
||
}
|
||
// if it has a link to dmoz.org then we are probably a dmoz mirror
|
||
// because we are required to have a link to dmoz if a mirror
|
||
bool checkForDmoz = true;
|
||
bool checkForAmazon = true;
|
||
if ( linkee &&
|
||
linkee->getDomainLen() == 8 &&
|
||
linkee->getDomain() )
|
||
if ( strncmp ( linkee->getDomain() , "dmoz.org" , 8 ) == 0 ||
|
||
strncmp ( linkee->getDomain() , "dmoz.com" , 8 ) == 0 )
|
||
checkForDmoz = false;
|
||
|
||
if ( linker &&
|
||
linker->getDomainLen() == 8 &&
|
||
linker->getDomain() )
|
||
if ( strncmp ( linker->getDomain() , "dmoz.org" , 8 ) == 0 ||
|
||
strncmp ( linker->getDomain() , "dmoz.com" , 8 ) == 0 )
|
||
checkForDmoz = false;
|
||
|
||
// does the url of the linker have a dmoz path in it?
|
||
char *zstart = linker->getPath();
|
||
long zlen = linker->getPathLen();
|
||
char *zend = zstart + zlen;
|
||
// start at the end of the path
|
||
char *z = zend-1;
|
||
// back up to previous /
|
||
for ( ; z > zstart && *z != '/' ; z-- );
|
||
// make that the new end
|
||
zend = z + 1;
|
||
// need at least 2 path components before checking... keep count
|
||
long zcount = 2;
|
||
// begin the loop
|
||
while ( checkForDmoz && z > zstart ) {
|
||
// . backup until we hit the previous /
|
||
for ( z-- ; z > zstart && *z != '/' ; z-- );
|
||
// debug
|
||
//char tmp[2000];
|
||
//memcpy(tmp,z,zend-z);
|
||
//tmp[zend-z]=0;
|
||
//log("build: path=%s",tmp);
|
||
// look it up
|
||
// "/Arts/" --> 1
|
||
if ( --zcount > 0 ) continue;
|
||
if ( g_categories->getIndexFromPath (z, zend-z) < 0 ) continue;
|
||
*note = "has dmoz path";
|
||
return true;
|
||
}
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
// if it has a link to amazon.com
|
||
// because we are required to have a link to dmoz if a mirror
|
||
if ( linkee && linkee->getDomain() &&
|
||
strncmp ( linkee->getDomain(), "amazon.com" , 10 ) == 0 )
|
||
checkForAmazon = false;
|
||
|
||
if ( linker && linker->getDomain() &&
|
||
strncmp ( linker->getDomain(), "amazon.com" , 10 ) == 0 )
|
||
checkForAmazon = false;
|
||
|
||
// guestbook in hostname - domain?
|
||
char *hd = linker->getHost();
|
||
char *hd2 = linker->getDomain();
|
||
long hdlen = hd2 - hd;
|
||
if ( hd && hd2 && hdlen < 30 ) {
|
||
char c = hd[hdlen];
|
||
hd[hdlen] = '\0';
|
||
bool hasIt = false;
|
||
if ( strstr ( hd , "guestbook" ) ) hasIt = true;
|
||
hd[hdlen] = c;
|
||
if ( hasIt ) {
|
||
*note = "guestbook in hostname";
|
||
return true;
|
||
}
|
||
}
|
||
|
||
// do not allow any cgi url to vote
|
||
if ( linker->isCgi() ) { *note = "path is cgi"; return true; }
|
||
|
||
// breathe
|
||
QUICKPOLL(niceness);
|
||
|
||
long plen = linker->getPathLen();
|
||
|
||
// if very spammy!!
|
||
// if ( spam
|
||
|
||
// if the page has just one rel=nofollow tag then we know they
|
||
// are not a guestbook
|
||
//if ( links->hasRelNoFollow() ) plen = 0;
|
||
if ( plen > 1 ) {
|
||
char *p = linker->getPath();
|
||
//char c = p[plen-1];
|
||
//p[plen-1] = '\0';
|
||
//bool val = false;
|
||
if ( strncasestr ( p , "guest",plen,5) ) {
|
||
*note = "path has guest" ; return true; }
|
||
else if ( strncasestr ( p , "cgi",plen,3) ) {
|
||
*note = "path has cgi" ; return true; }
|
||
else if ( strncasestr ( p , "gast",plen,4) ) {
|
||
*note = "path has gast" ; return true; }
|
||
// german
|
||
else if ( strncasestr ( p , "gaest",plen,5) ) {
|
||
*note = "path has gaest" ; return true; }
|
||
else if ( strncasestr ( p , "gbook",plen,5) ) {
|
||
*note = "path has gbook" ; return true; }
|
||
// vietnamese?
|
||
else if ( strncasestr ( p , "akobook",plen,7) ) {
|
||
*note = "path has akobook" ; return true; }
|
||
else if ( strncasestr ( p , "/gb",plen,3) ) {
|
||
*note = "path has /gb" ; return true; }
|
||
else if ( strncasestr ( p , "msg",plen,3 ) ) {
|
||
*note = "path has msg" ; return true; }
|
||
else if ( strncasestr ( p , "messag",plen,6) ) {
|
||
*note = "path has messag" ; return true; }
|
||
else if ( strncasestr ( p , "board",plen,5) ) {
|
||
*note = "path has board" ; return true; }
|
||
else if ( strncasestr ( p , "coment",plen,6) ) {
|
||
*note = "path has coment" ; return true; }
|
||
else if ( strncasestr ( p , "comment",plen,7) ) {
|
||
*note = "path has comment" ; return true; }
|
||
else if ( strncasestr ( p , "linktrader",plen,10) ) {
|
||
*note = "path has linktrader" ; return true; }
|
||
else if ( strncasestr ( p , "tradelinks",plen,10) ) {
|
||
*note = "path has tradelinks" ; return true; }
|
||
else if ( strncasestr ( p , "trade-links",plen,11) ) {
|
||
*note = "path has trade-links" ; return true; }
|
||
else if ( strncasestr ( p , "linkexchange",plen,12) ) {
|
||
*note = "path has linkexchange" ; return true; }
|
||
else if ( strncasestr ( p , "link-exchange",plen,13 ) ) {
|
||
*note = "path has link-exchange" ; return true; }
|
||
else if ( strncasestr ( p , "reciprocal-link",plen,15) ) {
|
||
*note = "path has reciprocal-link"; return true; }
|
||
else if ( strncasestr ( p , "reciprocallink",plen, 14) ) {
|
||
*note = "path has reciprocallink" ; return true; }
|
||
else if ( strncasestr ( p , "/trackbacks/",plen,12 ) ) {
|
||
*note = "path has /trackbacks/" ; return true; }
|
||
|
||
//if ( gb_strcasestr ( p , "archive" ) ) val = true;
|
||
//if ( gb_strcasestr ( p , ".asp" ) ) val = true;
|
||
//if ( gb_strcasestr ( p , ".aspx" ) ) val = true;
|
||
// these are mostly link exchange pages. no they are not!
|
||
//if ( gb_strcasestr ( p , "link" ) ) val = true;
|
||
//p[plen-1] = c;
|
||
//if ( val ) { *note = "cgi or guestbook url"; return true; }
|
||
}
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
// scan through the content as fast as possible
|
||
char *content = xml->getContent();
|
||
long contentLen = xml->getContentLen();
|
||
|
||
|
||
// does title contain "web statistics for"?
|
||
long tlen ;
|
||
char *title = xml->getString ( "title" , &tlen );
|
||
if ( title && tlen > 0 ) {
|
||
// normalize title into buffer, remove non alnum chars
|
||
char buf[256];
|
||
char *d = buf;
|
||
char *dend = buf + 250;
|
||
char *s = title;
|
||
char *send = title + tlen;
|
||
while ( d < dend && s < send ) {
|
||
// remove punct
|
||
if ( ! is_alnum_a(*s) ) { s++; continue; }
|
||
*d = to_lower_a ( *s );
|
||
d++;
|
||
s++;
|
||
}
|
||
*d = '\0';
|
||
// see if it matches some catch phrases
|
||
bool val = false;
|
||
if ( strstr (buf,"webstatisticsfor" )) val = true;
|
||
if ( strstr (buf,"webserverstatisticsfor")) val = true;
|
||
else if ( strstr (buf,"usagestatisticsfor" )) val = true;
|
||
else if ( strstr (buf,"siteusageby" )) val = true;
|
||
else if ( strstr (buf,"surfstatsloganal" )) val = true;
|
||
else if ( strstr (buf,"webstarterhelpstats" )) val = true;
|
||
else if ( strstr (buf,"sitestatistics" )) val = true;
|
||
if ( val ) { *note = "stats page"; return true; }
|
||
}
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
/////////////////////////////////////////////////////
|
||
//
|
||
// check content for certain keywords and phrases
|
||
//
|
||
/////////////////////////////////////////////////////
|
||
|
||
char *haystack = content;
|
||
long haystackSize = contentLen;
|
||
|
||
// get our page quality, it serves as a threshold for some algos
|
||
//char quality = tr->getNewQuality();
|
||
|
||
char *linkPos = NULL;
|
||
if ( linkNode >= 0 ) linkPos = xml->getNode ( linkNode );
|
||
|
||
//if ( strstr ( linker->getUrl() , "usa_apartments1.htm") ) {
|
||
// log("hey");
|
||
// sleep(7);
|
||
//}
|
||
|
||
// loop:
|
||
// do not call them "bad links" if our link occurs before any
|
||
// comment section. our link's position therefore needs to be known,
|
||
// that is why we pass in linkPos.
|
||
// "n" is the number it matches.
|
||
long n;
|
||
long numNeedles1 = sizeof(s_needles1)/sizeof(Needle);
|
||
bool hadPreMatch;
|
||
getMatches2 ( s_needles1 ,
|
||
numNeedles1 ,
|
||
haystack ,
|
||
haystackSize ,
|
||
linkPos ,
|
||
&n ,
|
||
false , // stopAtFirstMatch
|
||
&hadPreMatch ,
|
||
true , // save quicktables
|
||
niceness );
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
// see if we got a hit
|
||
for ( long i = 0 ; i < numNeedles1 ; i++ ) {
|
||
long need = 1;
|
||
// open.thumbshots.org needs multiple counts
|
||
if ( i == 0 ) need = 5;
|
||
if ( s_needles1[i].m_count < need ) continue;
|
||
*note = s_needles1[i].m_string;
|
||
return true;
|
||
}
|
||
|
||
// now check outlinks on the page for these substrings
|
||
haystack = links->getLinkBuf();
|
||
haystackSize = links->getLinkBufLen();
|
||
long numNeedles2 = sizeof(s_needles2)/sizeof(Needle);
|
||
getMatches2 ( s_needles2 ,
|
||
numNeedles2 ,
|
||
haystack ,
|
||
haystackSize ,
|
||
NULL , // linkPos,
|
||
NULL , // &n ,
|
||
false , // stopAtFirstMatch?
|
||
NULL , // hadPreMatch?
|
||
true , // save quicktables
|
||
niceness );
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
// see if we got a hit
|
||
for ( long i = 0 ; i < numNeedles2 ; i++ ) {
|
||
long need = 1;
|
||
// open.thumbshots.org needs multiple counts
|
||
//if ( i == 9 ) need = 5;
|
||
if ( s_needles2[i].m_count < need ) continue;
|
||
*note = s_needles2[i].m_string;
|
||
return true;
|
||
}
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
//skiplinks:
|
||
// check for certain post tag, indicative of a comment-friendly blog
|
||
// <form method=post ... action=*comments*cgi-bin>
|
||
// <form method="post"
|
||
// action="http://www.mydomain.com/cgi-bin/mt-comments.cgi"
|
||
// name="comments_form" ...>
|
||
// <form method=POST
|
||
// action="http://peaceaction.org/wboard/wwwboard.cgi">
|
||
long nn = xml->getNumNodes();
|
||
bool gotTextArea = false;
|
||
bool gotSubmit = false;
|
||
for ( long i=0; i < nn ; i++ ) {
|
||
// <textarea> tags are bad... but only if we have not
|
||
// matched "track" or whatever from above... check for that
|
||
// if you uncommment this... otherwise you disable all blogs!
|
||
// Only do this check if we did match a comment related phrase
|
||
// in s_needles1[] BUT it was BEFORE our outlink. That
|
||
// basically means that we do *not* recognize the format of
|
||
// the comment page and so therefore need to be more
|
||
// restrictive about allowing this page to vote.
|
||
if ( ! hadPreMatch ) {
|
||
// is it a <textarea> tag?
|
||
if ( xml->getNodeId ( i ) == TAG_TEXTAREA )
|
||
gotTextArea = true;
|
||
// is it an <input> tag?
|
||
long len = 0;
|
||
if ( xml->getNodeId ( i ) == TAG_INPUT &&
|
||
xml->getString(i,"submit",&len)) gotSubmit = true;
|
||
}
|
||
// check for script tag
|
||
/*
|
||
if ( xml->getNodeId(i) == TAG_SCRIPT && quality < 80 ) {
|
||
// <script src=blah.com/fileparse.js"
|
||
// type="text/javascript"> is used to hide google
|
||
// ads, so don't allow those pages to vote either
|
||
long slen; xml->getString(i,"src",&slen);
|
||
if ( slen > 0 ) { *note = "script src"; return true; }
|
||
}
|
||
*/
|
||
if ( xml->getNodeId ( i ) != TAG_FORM ) continue;
|
||
|
||
// get the method field of this base tag
|
||
long slen;
|
||
char *s = (char *) xml->getString(i,"method",&slen);
|
||
// if not thee, skip it
|
||
if ( ! s || slen <= 0 ) continue;
|
||
//if ( slen != 4 ) continue;
|
||
// if not a post, skip it
|
||
//if ( strncasecmp ( s , "post" , 4 ) ) continue;
|
||
// get the action url
|
||
s = (char *) xml->getString(i,"action",&slen);
|
||
if ( ! s || slen <= 0 ) continue;
|
||
char c = s[slen];
|
||
s[slen]='\0';
|
||
bool val = false;
|
||
// this is a bit too strong, but i'ev seen an action of
|
||
// "cgi-bin/mt-leaveone.cgi" so we can't rely on "mt-comment"
|
||
if ( strstr ( s , "comment" ) ) val = true;
|
||
else if ( strstr ( s , "/MT/" ) ) val = true;
|
||
else if ( strstr ( s , "/mt/" ) ) val = true;
|
||
// they can have these search boxes though
|
||
if ( val && strstr ( s , "/mt/mt-search" ) ) val = false;
|
||
//else if ( strstr ( s , "cgi" ) ) val = true;
|
||
// eliminate some false positives
|
||
//if ( val && strstr ( s , "search" ) ) val = false;
|
||
s[slen] = c;
|
||
if ( val ) { *note = "post page"; return true; }
|
||
}
|
||
|
||
if ( gotTextArea && gotSubmit ) {
|
||
*note = "textarea tag";
|
||
return true;
|
||
}
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
// edu, gov, etc. can have link chains
|
||
if ( tldLen >= 3 && strncmp ( tld, "edu" , 3) == 0 ) return false;
|
||
if ( tldLen >= 3 && strncmp ( tld, "gov" , 3) == 0 ) return false;
|
||
|
||
// if linker is naughty, he cannot vote
|
||
if ( linker->isSpam() ) return true;
|
||
|
||
// if being called from PageTitledb.cpp for displaying a titlerec,
|
||
// then do not call this, because no linkee is provided in that case.
|
||
if ( !linkee ) {
|
||
*note = "linkee not found";
|
||
return false;//true;
|
||
}
|
||
|
||
// . allow sites in dmoz to have it too
|
||
// . no, there are too many spam porn sites in dmoz
|
||
long *ids = indCatIds;
|
||
long nids = numIndCatIds;
|
||
//if ( tr ) ids = indCatIds;//tr->getIndCatids();
|
||
//if ( tr ) nids = numIndCatids;//tr->getNumIndCatids();
|
||
for ( long j = 0 ; j < nids ; j++ )
|
||
if ( g_categories && g_categories->isIdAdult ( ids[j] ) ) {
|
||
*note = "dmoz porn";
|
||
return true;
|
||
}
|
||
|
||
QUICKPOLL( niceness );
|
||
// . if they link to any adult site, consider them link spam
|
||
// . just consider a 100 link radius around linkNode
|
||
long nl = links->getNumLinks();
|
||
long linkNum =links->findLinkNum(linkee->getUrl(),linkee->getUrlLen());
|
||
long i0 = linkNum - 100;
|
||
long i1 = linkNum + 100;
|
||
if ( i0 < 0 ) i0 = 0;
|
||
if ( i1 > nl ) i1 = nl;
|
||
for ( long i = i0 ; i < i1 ; i++ ) {
|
||
if ( ! g_categories ) continue;
|
||
Url uu; uu.set ( links->getLink(i), links->getLinkLen(i) );
|
||
char *h = uu.getUrl();
|
||
// include the trailing /
|
||
char *hend = uu.getHost() + uu.getHostLen() + 1;
|
||
unsigned long hhash = hash32 ( h , hend - h );
|
||
if ( g_categories->isInBadCat ( hhash ) ) {
|
||
*note = "links to dmoz filth";
|
||
log(LOG_DEBUG,"build: %s is filthy.",uu.getUrl());
|
||
return true;
|
||
}
|
||
QUICKPOLL( niceness );
|
||
|
||
// . is it near sporny links? (naughty domains or lotsa -'s)
|
||
// . if we are in a list of ads, chances are good the true
|
||
// nature of the ads will emerge...
|
||
if ( i == linkNum ) continue;
|
||
if ( i - linkNum >= 4 ) continue;
|
||
if ( i - linkNum <= -4 ) continue;
|
||
if ( ! uu.isSpam() ) continue;
|
||
*note = "near sporny outlink";
|
||
log(LOG_DEBUG,"build: %s is sporny.",uu.getUrl());
|
||
return true;
|
||
}
|
||
|
||
|
||
// . allow sites in dmoz to have it too
|
||
// . no, there are too many spam porn sites in dmoz
|
||
//if ( tr->getNumIndCatids() || tr->getNumCatids() ) return false;
|
||
|
||
// init these before the loop
|
||
long hlen = linkee->getHostLen();
|
||
char *host = linkee->getHost();
|
||
char *uu = linkee->getUrl();
|
||
char *uuend = host + hlen;
|
||
long uulen = uuend - uu;
|
||
long x = linkNode;
|
||
loop:
|
||
|
||
QUICKPOLL( niceness );
|
||
|
||
// return true right away if it is a link chain
|
||
//if ( tr->getDocQuality() < 60 &&
|
||
if ( siteNumInlinks < 1000 &&
|
||
isLinkChain ( xml , linker, linkee , x , note ) )
|
||
return true;
|
||
|
||
// if no domain, that's it
|
||
if ( ! uu || uulen <= 0 ) return false;
|
||
|
||
// . see if this domain is linked to in other areas of the document.
|
||
// . if any of those areas are not link chains, then assume we are
|
||
// not a link chain
|
||
for ( x++ ; x < nl ; x++ ) {
|
||
char *link = links->getLink (x);
|
||
long linkLen = links->getLinkLen (x);
|
||
if ( ! link ) continue;
|
||
if ( linkLen <= 0 ) continue;
|
||
if ( linkLen > uulen ) continue;
|
||
if ( strncmp ( link , uu , uulen ) ) continue;
|
||
// got a match, is it a link chain? if not, them we are not
|
||
goto loop;
|
||
}
|
||
|
||
return false;
|
||
|
||
}
|
||
|
||
// Criteria for being a link chain:
|
||
//
|
||
// 1. the "linkee" is in a chain of outlinks to external domains
|
||
// 2. all outlinks to the same hostname as "linkee" are in link chains
|
||
// 3. no plain text is present between "linkee" and one of the other
|
||
// outlinks in the chain
|
||
// 4. this might hurt blogrolls, and resource pages, but such links
|
||
// are kind of low quality anyway.
|
||
bool isLinkChain ( Xml *xml , Url *linker , Url *linkee , long linkNode ,
|
||
char **note ) {
|
||
|
||
//log(LOG_DEBUG,"build: doing %s",linker->m_url);
|
||
|
||
// if the linkee is internal (by domain) then not a link chain
|
||
if ( linkee->m_dlen == linker->m_dlen &&
|
||
strncmp ( linkee->m_domain , linker->m_domain,linkee->m_dlen)==0)
|
||
return false;
|
||
|
||
char *linkPos = NULL;
|
||
if ( linkNode >= 0 ) linkPos = xml->getNode ( linkNode );
|
||
|
||
// did we have text to the left/right of this link and after/before
|
||
// the neighboring link? assume not.
|
||
bool leftText = false;
|
||
bool rightText = false;
|
||
|
||
// the links on the left and right
|
||
Url leftUrl;
|
||
Url rightUrl;
|
||
bool leftMalformed = false;
|
||
bool rightMalformed = false;
|
||
// these do not have constructors so we must reset them
|
||
leftUrl.reset();
|
||
rightUrl.reset();
|
||
long i ;
|
||
// . see if we are alone in a table or not
|
||
// . table must occur before/after our left/right neighbor link
|
||
bool tableLeft = false;
|
||
bool tableRight = false;
|
||
|
||
// going backwards from linkNode we are not in a link
|
||
bool inLink = false;
|
||
|
||
// get the start of an anchor tag on our immediate left
|
||
for ( i = linkNode - 1 ; i >= 0 ; i-- ) {
|
||
// do not look too far
|
||
if ( linkPos - xml->getNode(i) >= 1500 ) break;
|
||
if ( linkNode - i >= 90 ) break;
|
||
// NOTE: if you add more tags to this list, then also add
|
||
// to Vector::setPairHashes() as well
|
||
// stop at <title> or </title> tags
|
||
if ( xml->getNodeId(i) == TAG_TITLE ) break;
|
||
// stop at <ul> or </ul> tags
|
||
// no, otherwise, these lists are always "link chain left"
|
||
//if ( xml->getNodeId(i) == TAG_UL ) break;
|
||
// stop at <table> or </table> tags
|
||
if ( xml->getNodeId(i) == TAG_TABLE ) {
|
||
if ( ! xml->isBackTag(i) ) tableLeft = true;
|
||
break;
|
||
}
|
||
|
||
// check for *plain* text
|
||
if ( ! inLink && xml->getNodeId(i) == TAG_TEXTNODE ) {
|
||
// get the node as a string
|
||
char *p = xml->getNode(i);
|
||
char *pend = p + xml->getNodeLen(i);
|
||
// check for elipsis, that is a sign that we are a serp
|
||
for ( char *s = p ; s+2 < pend ; s++ ) {
|
||
//if ( is_alnum(*s) ) break;
|
||
if ( *s != '.' ) continue;
|
||
s++;
|
||
if ( *s != '.' ) continue;
|
||
s++;
|
||
if ( *s != '.' ) continue;
|
||
// ok, got it
|
||
*note = "search result right";
|
||
return true;
|
||
}
|
||
// if we already got text, but searching still for ...
|
||
if ( leftText ) continue;
|
||
// does it have alnum
|
||
if ( ! has_alpha_utf8 ( p , pend ) ) continue;
|
||
leftText = true;
|
||
// do not break yet, cont search for ellipsis!
|
||
}
|
||
|
||
// keep chugging if not an anchor tag, <a> or </a>
|
||
if ( xml->getNodeId(i) != TAG_A ) continue;
|
||
// if we are </a> then we are now in a link since we are moving
|
||
// backwards
|
||
if ( xml->isBackTag(i) ) { inLink = true; continue; }
|
||
// if we hit a forward tag and inLink was false... we had
|
||
// no corresponding back tag, so disconsider any text
|
||
if ( ! inLink ) rightText = false;
|
||
// no longer in an <a> tag
|
||
inLink = false;
|
||
|
||
// ok, get the url from this anchor tag
|
||
long ulen = 0;
|
||
char *u = (char *) xml->getString ( i, "href", &ulen );
|
||
// if we did not get one, that means it could have been
|
||
// malformed... like the href had a quote right b4 it
|
||
if ( ulen == 0 ) leftMalformed = true;
|
||
// normalize
|
||
if ( ulen > 0 )
|
||
leftUrl.set ( linker , u , ulen );
|
||
// . if NOT from the same domain, break out, otherwise continue
|
||
// . this helps us find the <table> tag in ad tables with
|
||
// multiple links to the same domain
|
||
// . this helps us accept a list of links to the same domain if
|
||
// there is left/right text, like the guy that had a list
|
||
// to 3 different gigablast.com links in a row with no
|
||
// text in between
|
||
if ( leftUrl.m_dlen != linkee->m_dlen ) break;
|
||
if ( strncmp ( leftUrl.m_domain ,
|
||
linkee->m_domain ,
|
||
linkee->m_dlen ) ) break;
|
||
}
|
||
|
||
// we start off in link text, since linkNode is an <a> tag
|
||
inLink = true;
|
||
// now loop through all the nodes after us
|
||
for ( i = linkNode + 1 ; i < xml->getNumNodes() ; i++ ) {
|
||
// stop if we've gone too far
|
||
if ( xml->getNode(i) - linkPos >= 1580 ) break;
|
||
if ( i - linkNode >= 95 ) break;
|
||
// stop at <title> or </title> tags
|
||
if ( xml->getNodeId(i) == TAG_TITLE ) break;
|
||
// stop at <table> or </table> tags
|
||
if ( xml->getNodeId(i) == TAG_TABLE ) {
|
||
// note it for table ads
|
||
if ( xml->isBackTag(i) ) tableRight = true;
|
||
break;
|
||
}
|
||
|
||
// check for *plain* text
|
||
if ( ! inLink && xml->getNodeId(i) == TAG_TEXTNODE ) {
|
||
// get the node as a string
|
||
char *p = xml->getNode(i);
|
||
char *pend = p + xml->getNodeLen(i);
|
||
// check for elipsis, that is a sign that we are a serp
|
||
for ( char *s = p ; s+2 < pend ; s++ ) {
|
||
//if ( is_alnum(*s) ) break;
|
||
if ( *s != '.' ) continue;
|
||
s++;
|
||
if ( *s != '.' ) continue;
|
||
s++;
|
||
if ( *s != '.' ) continue;
|
||
// ok, got it
|
||
*note = "search result right";
|
||
return true;
|
||
}
|
||
// if we already got text, but searching still for ...
|
||
if ( rightText ) continue;
|
||
// does it have alnum
|
||
if ( ! has_alpha_utf8 ( p , pend ) ) continue;
|
||
rightText = true;
|
||
// do not break yet, cont search for ellipsis!
|
||
}
|
||
|
||
// keep chugging if not an anchor tag, <a> or </a>
|
||
if ( xml->getNodeId(i) != TAG_A ) continue;
|
||
// skip if not a forward tag
|
||
if ( xml->isBackTag(i) ) { inLink = false; continue; }
|
||
// we are now in a link
|
||
inLink = true;
|
||
// stop text here
|
||
//stopTextScan = i;
|
||
// ok, get the url
|
||
long ulen = 0;
|
||
char *u = (char *) xml->getString ( i, "href", &ulen );
|
||
// if we did not get one, that means it could have been
|
||
// malformed... like the href had a quote right b4 it
|
||
if ( ulen == 0 ) rightMalformed = true;
|
||
// normalize
|
||
if ( ulen > 0 )
|
||
rightUrl.set ( linker, u , ulen );
|
||
// . if NOT from the same domain, break out, otherwise continue
|
||
// . this helps us find the <table> tag in ad tables with
|
||
// multiple links to the same domain
|
||
// . this helps us accept a list of links to the same domain if
|
||
// there is left/right text, like the guy that had a list
|
||
// to 3 different gigablast.com links in a row with no
|
||
// text in between
|
||
if ( rightUrl.m_dlen != linkee->m_dlen ) break;
|
||
if ( strncmp ( rightUrl.m_domain ,
|
||
linkee->m_domain ,
|
||
linkee->m_dlen ) ) break;
|
||
}
|
||
|
||
if ( tableLeft && tableRight ) {
|
||
*note = "ad table";
|
||
return true;
|
||
}
|
||
|
||
// if we had text on both sides of us, we are not a link chain
|
||
if ( leftText && rightText ) return false;
|
||
|
||
if ( ! leftText && rightText ) *note = "link chain left";
|
||
else if ( ! rightText && leftText ) *note = "link chain right";
|
||
else *note = "link chain middle";
|
||
|
||
return true;
|
||
}
|
||
|
||
/*
|
||
//we want to find the position of the linkee in the linker's xml class
|
||
//if it is in a series of links without vertical space in the end of
|
||
//the document, then we won't count it
|
||
bool isLinkSpam2 ( Url *linkee, TitleRec *tr, Xml *xml, Links *links ,
|
||
long maxDocLen ) {
|
||
long linkNum = links->findLinkNum(linkee->getUrl(),
|
||
linkee->getUrlLen());
|
||
if(linkNum < 0) return false;
|
||
|
||
long thisXmlNode;
|
||
long nextXmlNode;
|
||
long numInARow = 1;
|
||
long needInARow = 3;
|
||
|
||
long thisLink = linkNum;
|
||
long nextLink = thisLink + 1;
|
||
short nodeId;
|
||
long i;
|
||
long len;
|
||
char *c;
|
||
char *cend;
|
||
|
||
goRight:
|
||
if(nextLink >= links->getNumLinks()) {
|
||
//we can only go left from here.
|
||
//log(LOG_WARN, "links: endotheline");
|
||
goto goLeft;
|
||
}
|
||
thisXmlNode = links->getNodeNum(thisLink);
|
||
nextXmlNode = links->getNodeNum(nextLink);
|
||
|
||
//skip link text
|
||
for ( i=thisXmlNode; i < nextXmlNode ; i++ ) {
|
||
if(xml->isBackTag(i)) break;
|
||
}
|
||
//could we find link text close tag?
|
||
if(i == nextXmlNode) {
|
||
//log(LOG_WARN, "links: couldn't find it");
|
||
goto goLeft;
|
||
}
|
||
for (; i < nextXmlNode ; i++ ) {
|
||
nodeId = xml->getNodeId(i);
|
||
if(nodeId == TAG_BR ||//br
|
||
nodeId == TAG_TR ||//tr
|
||
nodeId == TAG_LI) {//li
|
||
//log(LOG_WARN, "links: breaking tag");
|
||
goto goLeft;
|
||
}
|
||
//do we have a rel="nofollow" link which isn't in the
|
||
//links class. If so, skip past it's link text.
|
||
if(nodeId == TAG_A) {
|
||
for ( ; i < nextXmlNode ; i++ ) {
|
||
if(xml->isBackTag(i)) break;
|
||
}
|
||
if(i == nextXmlNode) goto goLeft;
|
||
}
|
||
|
||
if(nodeId != TAG_TEXTNODE && nodeId != TAG_B) continue;
|
||
c = xml->getNode(i);
|
||
len = xml->getNodeLen(i);
|
||
cend = c + len;
|
||
|
||
while (c < cend) {
|
||
if(*c == '&')
|
||
while (c < cend && *c != ';') c++;
|
||
if(is_alnum_utf8(c)) {
|
||
//log(LOG_WARN, "links: stupid %c", *c);
|
||
goto goLeft;
|
||
}
|
||
c++;
|
||
}
|
||
}
|
||
|
||
|
||
//ok, so we got to the next link without
|
||
//breaking, try getting the next link
|
||
numInARow++;
|
||
if(numInARow >= needInARow) goto goLeft;
|
||
|
||
thisLink++;
|
||
nextLink++;
|
||
goto goRight;
|
||
|
||
goLeft:
|
||
//log(LOG_WARN, "links: numinarow %li linkee %s , linker %s",
|
||
//numInARow, linkee->getUrl(), tr->getUrl()->getUrl());
|
||
if(numInARow >= needInARow) return true;
|
||
|
||
//get out if we've already tried going left
|
||
if(thisLink < linkNum) return false;
|
||
|
||
needInARow = needInARow - numInARow;
|
||
thisLink = linkNum - needInARow;
|
||
nextLink = thisLink + 1;
|
||
if(thisLink < 0) return false;
|
||
goto goRight;
|
||
|
||
|
||
return false;
|
||
}
|
||
*/
|
||
|
||
static Needle s_needles3[] = {
|
||
// this often directly precedes the comment section
|
||
{"[trackback" , 0 , 1 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"comtext" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"comment" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"coment" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"trackback" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"ping" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"followup" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"class=\"response" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
// this can signify a blog entry, not just a comment
|
||
//{"class=\"entry" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
// these seem to be more indicative of posted comments
|
||
{"class=\"posted" , 0 , 8 , 1 , 0 , NULL , 0 },
|
||
{"id=\"posted" , 0 , 8 , 1 , 0 , NULL , 0 },
|
||
{"name=\"posted" , 0 , 8 , 1 , 0 , NULL , 0 },
|
||
// annoying little textbox thingy
|
||
{"class=\"shoutbox" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"id=\"comment" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"id=\"coment" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"id=\"trackback" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"id=\"ping" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"id=\"followup" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"id=\"response" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"name=\"comment" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"name=\"coment" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"name=\"trackback" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"name=\"ping" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"name=\"followup" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
{"name=\"response" , 0 , 8 , 1 , 0 , NULL , 0 , NULL } ,
|
||
// message boards
|
||
{"anonymous user" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"anonymer user" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"date posted" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"post your notice" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
//{"edit this page" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
// edit</a><br>
|
||
{"edit<a]br" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
// link to edit a comment
|
||
{">edit</a" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
// these often indicate blog entries, not just comments
|
||
//{"reply with quote" , 0 , 9 , 0 , 0 , NULL , 0 , NULL } ,
|
||
//{">post a reply" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
//{"post reply" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
//{"submit post" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
//{">post message" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
//{">post a comment" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
//{">leave a comment" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
//{">post comments" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
// Comments</font> (0) after each posted entry...
|
||
//{">comments<" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"comments: <" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"comments:<" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
//{"comment:" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"reacties:" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"comentarios:" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"coment<EFBFBD>rios:" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{">message:" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">mensagem:" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">faca seu comentario" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">fa<66>a seu coment<6E>rio" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
// comment add in german
|
||
{">Kommentar hinzuf" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">add my comment" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
// title of the text area box
|
||
{">your comment" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"your comment<" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">comment by" , 0 , 10, 1 , 0 , NULL , 0 , NULL } ,
|
||
{">scrivi un commento" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{">scrivi il tuo commento" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"add comment" , 0 , 10, 0 , 0 , NULL , 0 , NULL } ,
|
||
{"trackbacks for the art" , 0 , 12, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"these trackbacks have been re", 0 , 13, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"trackback pings" , 0 , 13, 1 , 0 , NULL , 0 , NULL } ,
|
||
{"read the rest of this com" , 0 , 13, 1 , 0 , NULL , 0 , NULL } ,
|
||
// that was the opinion of ...
|
||
{"das war die meinung von" , 0 , 13, 1 , 0 , NULL , 0 , NULL } ,
|
||
//{"add new comment" , 0 , 14, 0 , 0 , NULL , 0 },
|
||
//{"add message" , 0 , 14, 0 , 0 , NULL , 0 },
|
||
// tagboard software allows free submits. it has this in
|
||
// an html comment tag...
|
||
{"2002 natali ardianto" , 0 , 14, 0 , 0 , NULL , 0 , NULL } ,
|
||
// guestbooks
|
||
//{"guestbook</title" , 0 , 13, 0 , 0 , NULL , 0 , NULL } ,
|
||
//{"gastenboek</title" , 0 , 13, 0 , 0 , NULL , 0 , NULL } ,
|
||
};
|
||
|
||
// gives you where the comment section starts
|
||
// looks for only the first comment section
|
||
char *getCommentSection ( char *haystack ,
|
||
long haystackSize ,
|
||
long niceness ){
|
||
|
||
// get our page quality, it serves as a threshold for some algos
|
||
//char quality = tr->getNewQuality();
|
||
|
||
//char *linkPos = NULL;
|
||
//if ( linkNode >= 0 ) linkPos = xml->getNode ( linkNode );
|
||
|
||
//if ( strstr ( linker->getUrl() , "usa_apartments1.htm") ) {
|
||
// log("hey");
|
||
// sleep(7);
|
||
//}
|
||
|
||
// loop:
|
||
// do not call them "bad links" if our link occurs before any
|
||
// comment section. our link's position therefore needs to be known,
|
||
// that is why we pass in linkPos.
|
||
// "n" is the number it matches.
|
||
//long numNeedles1 = sizeof(s_needles3)/sizeof(Needle);
|
||
return getMatches2 ( s_needles3 ,
|
||
sizeof(s_needles3)/sizeof(Needle),
|
||
haystack ,
|
||
haystackSize,
|
||
NULL ,
|
||
NULL ,
|
||
true ,// stopAtFirstMatch
|
||
NULL , // hadPreMatch?
|
||
false , // save quick tables?
|
||
niceness );
|
||
}
|