mosesdecoder/misc/merge-sorted.cc

// This program takes gzipped sorted files and merges them in sorted order
// to stdout. Written by Ulrich Germann
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/shared_ptr.hpp>
#include <algorithm>
#include <string>
#include <vector>
#include "moses/TranslationModel/UG/generic/file_io/ug_stream.h"
using namespace std;
using namespace ugdiss;
using namespace boost::iostreams;

typedef boost::shared_ptr<filtering_istream> fptr;

class Part
{
  string       fname;
  fptr             f;
  string my_lines[2];
  size_t         ctr;
public:
  string const& line() const 
  { 
    static string empty_line;
    return f ? my_lines[ctr%2] : empty_line; 
  }

  Part(string _fname) : ctr(0)
  {
    fname = _fname;
    f.reset(open_input_stream(fname));
    if (!getline(*f, my_lines[0])) f.reset();
  }

  bool next() 
  {
    if (!f) return false;
    if (!getline(*f, my_lines[++ctr%2]))
      {
        f.reset();
        --ctr;
        return false;
      }
     assert(my_lines[(ctr-1)%2] <= my_lines[ctr%2]);
    return true;
  }

  bool operator <(Part const& other) const 
  { return line() < other.line(); }

  bool operator <=(Part const& other) const 
  { return line() <= other.line(); }

  bool operator >(Part const& other) const 
  { return line() > other.line(); }

  bool operator >=(Part const& other) const 
  { return line() >= other.line(); }

  bool go(ostream& out)
  {
    if (!f) return false;
#if 0
    if (ctr)
      {
        out << fname << "-" << ctr - 1 << "-";
        out << my_lines[(ctr - 1)%2] << endl;
      }
    do 
      {
        out << fname << " " << ctr << " ";
        out << line() << "\n";
      }
    while (next() && my_lines[0] == my_lines[1]);
#else
    do    { out << line() << "\n"; } 
    while (next() && my_lines[0] == my_lines[1]);
    out.flush();
#endif
    return f != NULL;
  }
  
};


int main(int argc, char* argv[])
{
  vector<Part> parts;
  for (int i = 1; i < argc; ++i)
    parts.push_back(Part(argv[i]));
  make_heap(parts.begin(), parts.end(), greater<Part>());
  while (parts.size())
    {
      pop_heap(parts.begin(), parts.end(), greater<Part>());
      if (parts.back().go(cout))
        push_heap(parts.begin(), parts.end(), greater<Part>());
      else parts.pop_back();
    }
}