Merge pull request #2046 from lonvia/less-parallel-ranking

Only index larger batches for rank 30
This commit is contained in:
Sarah Hoffmann 2020-11-06 09:39:07 +01:00 committed by GitHub
commit 66595c2d2b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -135,19 +135,19 @@ class Indexer(object):
for rank in range(max(1, self.minrank), self.maxrank): for rank in range(max(1, self.minrank), self.maxrank):
self.index(RankRunner(rank)) self.index(RankRunner(rank))
if self.maxrank == 30: if self.maxrank == 30:
self.index(RankRunner(0)) self.index(RankRunner(0))
self.index(InterpolationRunner(), 20) self.index(InterpolationRunner(), 20)
self.index(RankRunner(self.maxrank), 20)
self.index(RankRunner(self.maxrank), 20) else:
self.index(RankRunner(self.maxrank))
def index(self, obj, batch=1): def index(self, obj, batch=1):
""" Index a single rank or table. `obj` describes the SQL to use """ Index a single rank or table. `obj` describes the SQL to use
for indexing. `batch` describes the number of objects that for indexing. `batch` describes the number of objects that
should be processed with a single SQL statement should be processed with a single SQL statement
""" """
log.warning("Starting {}".format(obj.name())) log.warning("Starting %s (using batch size %s)", obj.name(), batch)
cur = self.conn.cursor() cur = self.conn.cursor()
cur.execute(obj.sql_count_objects()) cur.execute(obj.sql_count_objects())