mirror of
https://github.com/browsermt/bergamot-translator.git
synced 2024-09-11 05:35:33 +03:00
Bunch of integers on containers to size_ts
This commit is contained in:
parent
d7556bc168
commit
0296a38cd4
@ -16,7 +16,7 @@ Batcher::Batcher(Ptr<Options> options) {
|
||||
}
|
||||
|
||||
void Batcher::addSentenceWithPriority(RequestSentence &sentence) {
|
||||
int bucket_id = sentence.numTokens();
|
||||
size_t bucket_id = sentence.numTokens();
|
||||
assert(bucket_id < bucket_.size());
|
||||
bucket_[bucket_id].insert(sentence);
|
||||
}
|
||||
@ -29,9 +29,9 @@ bool Batcher::cleaveBatch(Batch &batch) {
|
||||
// implementation should at least be as fast as marian's maxi-batch with full
|
||||
// corpus size as maxi-batch size.
|
||||
batch.clear();
|
||||
int paddedBatchSize = 0;
|
||||
size_t paddedBatchSize = 0;
|
||||
|
||||
for (int length = 0; length < bucket_.size(); length++) {
|
||||
for (size_t length = 0; length < bucket_.size(); length++) {
|
||||
auto p = bucket_[length].begin();
|
||||
while (p != bucket_[length].end()) {
|
||||
paddedBatchSize = (batch.size() + 1) * length;
|
||||
@ -52,7 +52,7 @@ bool Batcher::cleaveBatch(Batch &batch) {
|
||||
}
|
||||
|
||||
void Batcher::addWholeRequest(Ptr<Request> request) {
|
||||
for (int i = 0; i < request->numSegments(); i++) {
|
||||
for (size_t i = 0; i < request->numSegments(); i++) {
|
||||
RequestSentence requestSentence(i, request);
|
||||
addSentenceWithPriority(requestSentence);
|
||||
}
|
||||
|
@ -30,9 +30,9 @@ public:
|
||||
bool operator>>(Batch &batch); // alias
|
||||
|
||||
private:
|
||||
unsigned int miniBatchWords;
|
||||
size_t miniBatchWords;
|
||||
std::vector<std::set<RequestSentence>> bucket_;
|
||||
unsigned int batchNumber_{0};
|
||||
size_t batchNumber_{0};
|
||||
};
|
||||
|
||||
} // namespace bergamot
|
||||
|
@ -1,7 +1,7 @@
|
||||
#include "request.h"
|
||||
#include "sentence_ranges.h"
|
||||
#include "definitions.h"
|
||||
#include "response.h"
|
||||
#include "sentence_ranges.h"
|
||||
|
||||
#include "common/logging.h"
|
||||
|
||||
@ -11,7 +11,7 @@ namespace marian {
|
||||
namespace bergamot {
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
Request::Request(unsigned int Id, int lineNumberBegin,
|
||||
Request::Request(size_t Id, size_t lineNumberBegin,
|
||||
std::vector<Ptr<Vocab const>> &vocabs, std::string &&source,
|
||||
Segments &&segments, SentenceRanges &&sourceRanges,
|
||||
std::promise<Response> responsePromise)
|
||||
|
@ -17,9 +17,9 @@
|
||||
#ifndef SRC_BERGAMOT_REQUEST_H_
|
||||
#define SRC_BERGAMOT_REQUEST_H_
|
||||
|
||||
#include "sentence_ranges.h"
|
||||
#include "definitions.h"
|
||||
#include "response.h"
|
||||
#include "sentence_ranges.h"
|
||||
|
||||
#include "common/logging.h"
|
||||
#include "data/types.h"
|
||||
@ -35,7 +35,7 @@ namespace bergamot {
|
||||
|
||||
class Request {
|
||||
public:
|
||||
Request(unsigned int Id, int lineNumberBegin,
|
||||
Request(size_t Id, size_t lineNumberBegin,
|
||||
std::vector<Ptr<Vocab const>> &vocabs_, std::string &&source,
|
||||
Segments &&segments, SentenceRanges &&sourceTokenRanges,
|
||||
std::promise<Response> responsePromise);
|
||||
@ -64,8 +64,8 @@ public:
|
||||
void completeRequest();
|
||||
|
||||
private:
|
||||
unsigned int Id_;
|
||||
int lineNumberBegin_;
|
||||
size_t Id_;
|
||||
size_t lineNumberBegin_;
|
||||
|
||||
// Multiple translation-workers can concurrently access the same Request. The
|
||||
// following atomic atomically operates on the variable holding sentences
|
||||
|
@ -83,17 +83,13 @@ std::future<Response> Service::translate(std::string &&input) {
|
||||
}
|
||||
|
||||
void Service::stop() {
|
||||
int counter = 0;
|
||||
for (auto &worker : workers_) {
|
||||
Batch poison = Batch::poison();
|
||||
pcqueue_.ProduceSwap(poison);
|
||||
++counter;
|
||||
}
|
||||
|
||||
counter = 0;
|
||||
for (auto &worker : workers_) {
|
||||
worker.join();
|
||||
++counter;
|
||||
}
|
||||
|
||||
workers_.clear(); // Takes care of idempotency.
|
||||
|
@ -45,8 +45,8 @@ public:
|
||||
~Service();
|
||||
|
||||
private:
|
||||
unsigned int requestId_;
|
||||
int numWorkers_;
|
||||
size_t requestId_;
|
||||
size_t numWorkers_;
|
||||
|
||||
// vocabs are used to construct a Request, which later uses it to construct
|
||||
// Response (decode from words to string).
|
||||
|
@ -1,7 +1,7 @@
|
||||
#include "text_processor.h"
|
||||
#include "sentence_ranges.h"
|
||||
#include "data/types.h"
|
||||
#include "definitions.h"
|
||||
#include "sentence_ranges.h"
|
||||
|
||||
#include "common/options.h"
|
||||
#include "data/vocab.h"
|
||||
@ -51,12 +51,12 @@ void TextProcessor::process(const string_view &query, Segments &segments,
|
||||
void TextProcessor::truncate(Segment &segment,
|
||||
std::vector<string_view> &wordRanges,
|
||||
Segments &segments, SentenceRanges &sourceRanges) {
|
||||
for (int offset = 0; offset < segment.size();
|
||||
for (size_t offset = 0; offset < segment.size();
|
||||
offset += max_input_sentence_tokens_) {
|
||||
auto start = segment.begin() + offset;
|
||||
|
||||
unsigned int left = segment.size() - offset;
|
||||
unsigned int diff = std::min(max_input_sentence_tokens_, left);
|
||||
size_t left = segment.size() - offset;
|
||||
size_t diff = std::min(max_input_sentence_tokens_, left);
|
||||
|
||||
segments.emplace_back(start, start + diff);
|
||||
segments.back().push_back(sourceEosId());
|
||||
|
@ -1,10 +1,10 @@
|
||||
#ifndef SRC_BERGAMOT_TEXT_PROCESSOR_H_
|
||||
#define SRC_BERGAMOT_TEXT_PROCESSOR_H_
|
||||
|
||||
#include "sentence_ranges.h"
|
||||
#include "data/types.h"
|
||||
#include "data/vocab.h"
|
||||
#include "definitions.h"
|
||||
#include "sentence_ranges.h"
|
||||
|
||||
#include "sentence_splitter.h"
|
||||
|
||||
@ -41,7 +41,7 @@ private:
|
||||
|
||||
std::vector<Ptr<Vocab const>> *vocabs_;
|
||||
SentenceSplitter sentence_splitter_;
|
||||
unsigned int max_input_sentence_tokens_;
|
||||
size_t max_input_sentence_tokens_;
|
||||
};
|
||||
|
||||
} // namespace bergamot
|
||||
|
Loading…
Reference in New Issue
Block a user