enso-org/enso
1
1
Fork 0
mirror of https://github.com/enso-org/enso.git synced 2024-12-26 16:52:16 +03:00
Code Issues Projects Releases Wiki Activity

Adding vectorized implementations to some Column operations (#10795)

Browse Source 
- Part of #6256 - implements operations that could have been vectorized without changes to the overall infrastructure
This commit is contained in:
Radosław Waśko 2024-08-13 10:53:39 +02:00 committed by GitHub
parent ff7e31c237
commit b1958f8aa3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
37 changed files with 1966 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
distribution/lib/Standard/Examples/0.0.0-dev/data/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
data_2500_rows.csv

7
distribution/lib/Standard/Table/0.0.0-dev/src/Column.enso
View File

@ -992,8 +992,11 @@ type Column
example_coalesce = Examples.decimal_column.coalesce Examples.integer_column
coalesce : (Any | Vector Any) -> Column
coalesce self values =
fallback a b = a.if_nothing b
run_vectorized_many_op self "coalesce" fallback values
vec = Vector.unify_vector_or_element values
new_name = self.naming_helper.function_name "coalesce" [self]+vec
result = if values.is_empty then self else
values.fold self acc-> v-> acc.fill_nothing v
result.rename new_name
## GROUP Standard.Base.Math
ICON transform4

7
distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso
View File

@ -161,8 +161,11 @@ type Bench
case get_benchmark_report_path of
Nothing -> Nothing
path ->
line = 'Label,Phase,"Invocations count","Average time (ms)","Time Stdev"'
line.write path on_existing_file=Existing_File_Behavior.Backup
f = File.new path
# Only write the header line if this is the first time writing to this report file.
if f.exists.not then
line = 'Label,Phase,"Invocations count","Average time (ms)","Time Stdev"'
line.write f on_existing_file=Existing_File_Behavior.Backup
self.fold Nothing _-> g-> s->
c = g.configuration

152
std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/GenericBinaryOpReturningBoolean.java Normal file
View File

@ -0,0 +1,152 @@
package org.enso.table.data.column.operation.map.bool;
import java.util.BitSet;
import org.enso.table.data.column.operation.map.BinaryMapOperation;
import org.enso.table.data.column.operation.map.MapOperationProblemAggregator;
import org.enso.table.data.column.storage.BoolStorage;
import org.enso.table.data.column.storage.SpecializedStorage;
import org.enso.table.data.column.storage.Storage;
import org.graalvm.polyglot.Context;
/**
* A generic binary operation that takes two values of some type T and returns a boolean.
*
* <p>If any of the two values is null, the result will also be null.
*/
public abstract class GenericBinaryOpReturningBoolean<T, S extends SpecializedStorage<T>>
extends BinaryMapOperation<T, S> {
public GenericBinaryOpReturningBoolean(String name) {
super(name);
}
/**
* Tries to cast an object to the type T.
*
* <p>Returns {@code null} if the types are not compatible.
*/
protected abstract T tryCast(Object object);
protected abstract boolean doOperation(T a, T b);
protected abstract boolean doOther(T a, Object b);
@Override
public Storage<?> runBinaryMap(
S storage, Object arg, MapOperationProblemAggregator problemAggregator) {
if (arg == null) {
return BoolStorage.makeEmpty(storage.size());
} else {
T argT = tryCast(arg);
if (argT != null) {
return runHomogenousMap(storage, argT);
} else {
return runMixedMap(storage, arg);
}
}
}
@Override
public Storage<?> runZip(
S storage, Storage<?> arg, MapOperationProblemAggregator problemAggregator) {
assert arg != null;
if (storage.getType().equals(arg.getType())
&& arg instanceof SpecializedStorage<?> argStorage) {
SpecializedStorage<T> argTStorage = storage.castIfSameType(argStorage);
assert argTStorage != null : "We checked that types are equal so cast should not fail";
return runHomogenousZip(storage, argTStorage);
} else {
return runMixedZip(storage, arg);
}
}
private BoolStorage runHomogenousMap(S storage, T arg) {
BitSet newVals = new BitSet();
BitSet newIsNothing = new BitSet();
Context context = Context.getCurrent();
int n = storage.size();
for (int i = 0; i < n; i++) {
if (storage.isNothing(i)) {
newIsNothing.set(i);
} else {
T storageItem = storage.getItemBoxed(i);
assert storageItem != null : "isNothing returned true but element was null";
boolean r = doOperation(storageItem, arg);
newVals.set(i, r);
}
context.safepoint();
}
return new BoolStorage(newVals, newIsNothing, n, false);
}
private BoolStorage runMixedMap(S storage, Object arg) {
BitSet newVals = new BitSet();
BitSet newIsNothing = new BitSet();
Context context = Context.getCurrent();
int n = storage.size();
for (int i = 0; i < n; i++) {
if (storage.isNothing(i)) {
newIsNothing.set(i);
} else {
T storageItem = storage.getItemBoxed(i);
assert storageItem != null : "isNothing returned true but element was null";
boolean r = doOther(storageItem, arg);
newVals.set(i, r);
}
context.safepoint();
}
return new BoolStorage(newVals, newIsNothing, n, false);
}
private BoolStorage runHomogenousZip(S storage, SpecializedStorage<T> argStorage) {
BitSet newVals = new BitSet();
BitSet newIsNothing = new BitSet();
Context context = Context.getCurrent();
int n = storage.size();
int m = argStorage.size();
for (int i = 0; i < n; i++) {
if (storage.isNothing(i) || !(i < m) || argStorage.isNothing(i)) {
newIsNothing.set(i);
} else {
T storageItem = storage.getItemBoxed(i);
T argItem = argStorage.getItemBoxed(i);
assert storageItem != null : "isNothing returned true but element was null";
assert argItem != null : "isNothing returned true but element was null";
boolean r = doOperation(storageItem, argItem);
newVals.set(i, r);
}
context.safepoint();
}
return new BoolStorage(newVals, newIsNothing, n, false);
}
private BoolStorage runMixedZip(S storage, Storage<?> argStorage) {
BitSet newVals = new BitSet();
BitSet newIsNothing = new BitSet();
Context context = Context.getCurrent();
int n = storage.size();
int m = argStorage.size();
for (int i = 0; i < n; i++) {
if (storage.isNothing(i) || !(i < m) || argStorage.isNothing(i)) {
newIsNothing.set(i);
} else {
T storageItem = storage.getItemBoxed(i);
Object argItem = argStorage.getItemBoxed(i);
assert storageItem != null : "isNothing returned true but element was null";
assert argItem != null : "isNothing returned true but element was null";
T argT = tryCast(argItem);
boolean r = (argT != null) ? doOperation(storageItem, argT) : doOther(storageItem, argItem);
newVals.set(i, r);
}
context.safepoint();
}
return new BoolStorage(newVals, newIsNothing, n, false);
}
}

26
std-bits/table/src/main/java/org/enso/table/data/column/operation/map/datetime/TimeLikeBinaryOpReturningBoolean.java Normal file
View File

@ -0,0 +1,26 @@
package org.enso.table.data.column.operation.map.datetime;
import org.enso.base.polyglot.Polyglot_Utils;
import org.enso.table.data.column.operation.map.bool.GenericBinaryOpReturningBoolean;
import org.enso.table.data.column.storage.SpecializedStorage;
public abstract class TimeLikeBinaryOpReturningBoolean<T>
extends GenericBinaryOpReturningBoolean<T, SpecializedStorage<T>> {
Class<T> clazz;
public TimeLikeBinaryOpReturningBoolean(String name, Class<T> objectType) {
super(name);
this.clazz = objectType;
}
@Override
protected T tryCast(Object object) {
// We need to adapt date/time values to ensure correct handling of polyglot values.
Object adapted = Polyglot_Utils.convertPolyglotValue(object);
if (clazz.isInstance(adapted)) {
return clazz.cast(adapted);
} else {
return null;
}
}
}

95
std-bits/table/src/main/java/org/enso/table/data/column/operation/map/datetime/TimeLikeCoalescingOperation.java Normal file
View File

@ -0,0 +1,95 @@
package org.enso.table.data.column.operation.map.datetime;
import org.enso.base.polyglot.Polyglot_Utils;
import org.enso.table.data.column.builder.Builder;
import org.enso.table.data.column.operation.map.BinaryMapOperation;
import org.enso.table.data.column.operation.map.MapOperationProblemAggregator;
import org.enso.table.data.column.storage.SpecializedStorage;
import org.enso.table.data.column.storage.Storage;
import org.enso.table.error.UnexpectedTypeException;
import org.graalvm.polyglot.Context;
public abstract class TimeLikeCoalescingOperation<T>
extends BinaryMapOperation<T, SpecializedStorage<T>> {
private final Class<T> inputTypeClass;
public TimeLikeCoalescingOperation(String name, Class<T> inputTypeClass) {
super(name);
this.inputTypeClass = inputTypeClass;
}
protected abstract Builder createOutputBuilder(int size);
protected abstract T doOperation(T a, T b);
@Override
public Storage<?> runBinaryMap(
SpecializedStorage<T> storage, Object arg, MapOperationProblemAggregator problemAggregator) {
int size = storage.size();
if (arg == null) {
return storage;
} else {
Object adapted = Polyglot_Utils.convertPolyglotValue(arg);
if (inputTypeClass.isInstance(adapted)) {
T casted = inputTypeClass.cast(adapted);
int n = storage.size();
Builder builder = createOutputBuilder(n);
Context context = Context.getCurrent();
for (int i = 0; i < size; i++) {
T r = storage.isNothing(i) ? casted : doOperation(storage.getItemBoxed(i), casted);
builder.appendNoGrow(r);
context.safepoint();
}
return builder.seal();
} else {
throw new UnexpectedTypeException(
"a " + inputTypeClass.getName() + " but got " + arg.getClass().getName());
}
}
}
@Override
public Storage<?> runZip(
SpecializedStorage<T> storage,
Storage<?> arg,
MapOperationProblemAggregator problemAggregator) {
if (arg.getType().equals(storage.getType())) {
if (arg instanceof SpecializedStorage<?> argStorage) {
SpecializedStorage<T> argTStorage = storage.castIfSameType(argStorage);
int n = storage.size();
Builder builder = createOutputBuilder(n);
Context context = Context.getCurrent();
for (int i = 0; i < n; i++) {
T a = storage.getItemBoxed(i);
T b = argTStorage.getItemBoxed(i);
T r;
if (a == null && b == null) {
r = null;
} else {
if (a == null) {
r = b;
} else if (b == null) {
r = a;
} else {
r = doOperation(a, b);
}
}
builder.appendNoGrow(r);
context.safepoint();
}
return builder.seal();
} else {
throw new IllegalArgumentException(
"Unexpected storage implementation for type "
+ storage.getType()
+ ": "
+ arg.getClass().getCanonicalName());
}
} else {
throw new UnexpectedTypeException("a " + storage.getType() + " column");
}
}
}

36
std-bits/table/src/main/java/org/enso/table/data/column/operation/map/numeric/arithmetic/MaxOp.java Normal file
View File

@ -0,0 +1,36 @@
package org.enso.table.data.column.operation.map.numeric.arithmetic;
import java.math.BigDecimal;
import java.math.BigInteger;
import org.enso.table.data.column.operation.map.MapOperationProblemAggregator;
import org.enso.table.data.column.storage.Storage;
public class MaxOp<T extends Number, I extends Storage<? super T>>
extends NumericBinaryOpCoalescing<T, I> {
public MaxOp() {
super(Storage.Maps.MAX);
}
@Override
public double doDouble(
double a, double b, int ix, MapOperationProblemAggregator problemAggregator) {
return Math.max(a, b);
}
@Override
public Long doLong(long a, long b, int ix, MapOperationProblemAggregator problemAggregator) {
return Math.max(a, b);
}
@Override
public BigInteger doBigInteger(
BigInteger a, BigInteger b, int ix, MapOperationProblemAggregator problemAggregator) {
return a.max(b);
}
@Override
public BigDecimal doBigDecimal(
BigDecimal a, BigDecimal b, int ix, MapOperationProblemAggregator problemAggregator) {
return a.max(b);
}
}

36
std-bits/table/src/main/java/org/enso/table/data/column/operation/map/numeric/arithmetic/MinOp.java Normal file
View File

@ -0,0 +1,36 @@
package org.enso.table.data.column.operation.map.numeric.arithmetic;
import java.math.BigDecimal;
import java.math.BigInteger;
import org.enso.table.data.column.operation.map.MapOperationProblemAggregator;
import org.enso.table.data.column.storage.Storage;
public class MinOp<T extends Number, I extends Storage<? super T>>
extends NumericBinaryOpCoalescing<T, I> {
public MinOp() {
super(Storage.Maps.MIN);
}
@Override
public double doDouble(
double a, double b, int ix, MapOperationProblemAggregator problemAggregator) {
return Math.min(a, b);
}
@Override
public Long doLong(long a, long b, int ix, MapOperationProblemAggregator problemAggregator) {
return Math.min(a, b);
}
@Override
public BigInteger doBigInteger(
BigInteger a, BigInteger b, int ix, MapOperationProblemAggregator problemAggregator) {
return a.min(b);
}
@Override
public BigDecimal doBigDecimal(
BigDecimal a, BigDecimal b, int ix, MapOperationProblemAggregator problemAggregator) {
return a.min(b);
}
}

274
std-bits/table/src/main/java/org/enso/table/data/column/operation/map/numeric/arithmetic/NumericBinaryOpCoalescing.java Normal file
View File

@ -0,0 +1,274 @@
package org.enso.table.data.column.operation.map.numeric.arithmetic;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.BitSet;
import org.enso.table.data.column.operation.map.MapOperationProblemAggregator;
import org.enso.table.data.column.operation.map.numeric.helpers.BigDecimalArrayAdapter;
import org.enso.table.data.column.operation.map.numeric.helpers.BigIntegerArrayAdapter;
import org.enso.table.data.column.operation.map.numeric.helpers.DoubleArrayAdapter;
import org.enso.table.data.column.storage.SpecializedStorage;
import org.enso.table.data.column.storage.Storage;
import org.enso.table.data.column.storage.numeric.AbstractLongStorage;
import org.enso.table.data.column.storage.numeric.BigDecimalStorage;
import org.enso.table.data.column.storage.numeric.BigIntegerStorage;
import org.enso.table.data.column.storage.numeric.DoubleStorage;
import org.enso.table.data.column.storage.numeric.LongStorage;
import org.graalvm.polyglot.Context;
/**
* A variant of NumericBinaryOpImplementation that has different null behaviour: if one of the
* values is null, the other non-null value is returned.
*/
public abstract class NumericBinaryOpCoalescing<T extends Number, I extends Storage<? super T>>
extends NumericBinaryOpImplementation<T, I> {
public NumericBinaryOpCoalescing(String name) {
super(name);
}
@Override
protected DoubleStorage runDoubleZip(
DoubleArrayAdapter a, DoubleArrayAdapter b, MapOperationProblemAggregator problemAggregator) {
Context context = Context.getCurrent();
int n = a.size();
int m = Math.min(a.size(), b.size());
long[] out = new long[n];
BitSet isNothing = new BitSet();
for (int i = 0; i < m; i++) {
boolean aNothing = a.isNothing(i);
boolean bNothing = b.isNothing(i);
if (aNothing && bNothing) {
isNothing.set(i);
} else {
double r;
if (aNothing) {
r = b.getItemAsDouble(i);
} else if (bNothing) {
r = a.getItemAsDouble(i);
} else {
r = doDouble(a.getItemAsDouble(i), b.getItemAsDouble(i), i, problemAggregator);
}
out[i] = Double.doubleToRawLongBits(r);
}
context.safepoint();
}
for (int i = m; i < n; ++i) {
if (a.isNothing(i)) {
isNothing.set(i);
} else {
out[i] = Double.doubleToRawLongBits(a.getItemAsDouble(i));
}
context.safepoint();
}
return new DoubleStorage(out, n, isNothing);
}
@Override
protected DoubleStorage runDoubleMap(
DoubleArrayAdapter a, Double b, MapOperationProblemAggregator problemAggregator) {
if (b == null) {
return a.intoStorage();
}
double bNonNull = b;
Context context = Context.getCurrent();
int n = a.size();
long[] out = new long[n];
BitSet isNothing = new BitSet();
for (int i = 0; i < n; i++) {
double r =
a.isNothing(i)
? bNonNull
: doDouble(a.getItemAsDouble(i), bNonNull, i, problemAggregator);
out[i] = Double.doubleToRawLongBits(r);
context.safepoint();
}
return new DoubleStorage(out, n, isNothing);
}
@Override
protected LongStorage runLongZip(
AbstractLongStorage a,
AbstractLongStorage b,
MapOperationProblemAggregator problemAggregator) {
Context context = Context.getCurrent();
int n = a.size();
int m = Math.min(a.size(), b.size());
long[] out = new long[n];
BitSet isNothing = new BitSet();
for (int i = 0; i < m; i++) {
boolean aNothing = a.isNothing(i);
boolean bNothing = b.isNothing(i);
if (aNothing && bNothing) {
isNothing.set(i);
} else {
if (aNothing) {
out[i] = b.getItem(i);
} else if (bNothing) {
out[i] = a.getItem(i);
} else {
Long r = doLong(a.getItem(i), b.getItem(i), i, problemAggregator);
if (r == null) {
isNothing.set(i);
} else {
out[i] = r;
}
}
}
context.safepoint();
}
for (int i = m; i < n; ++i) {
if (a.isNothing(i)) {
isNothing.set(i);
} else {
out[i] = a.getItem(i);
}
context.safepoint();
}
return new LongStorage(out, n, isNothing, INTEGER_RESULT_TYPE);
}
protected Storage<Long> runLongMap(
AbstractLongStorage a, Long b, MapOperationProblemAggregator problemAggregator) {
if (b == null) {
return a;
}
long bNonNull = b;
Context context = Context.getCurrent();
int n = a.size();
long[] out = new long[n];
BitSet isNothing = new BitSet();
for (int i = 0; i < n; i++) {
if (a.isNothing(i)) {
out[i] = bNonNull;
} else {
Long r = doLong(a.getItem(i), bNonNull, i, problemAggregator);
if (r == null) {
isNothing.set(i);
} else {
out[i] = r;
}
}
context.safepoint();
}
return new LongStorage(out, n, isNothing, INTEGER_RESULT_TYPE);
}
protected BigIntegerStorage runBigIntegerZip(
BigIntegerArrayAdapter a,
BigIntegerArrayAdapter b,
MapOperationProblemAggregator problemAggregator) {
Context context = Context.getCurrent();
int n = a.size();
int m = Math.min(a.size(), b.size());
BigInteger[] out = new BigInteger[n];
for (int i = 0; i < m; i++) {
BigInteger x = a.getItem(i);
BigInteger y = b.getItem(i);
if (x == null && y == null) {
out[i] = null;
} else {
if (x == null) {
out[i] = y;
} else if (y == null) {
out[i] = x;
} else {
BigInteger r = doBigInteger(x, y, i, problemAggregator);
out[i] = r;
}
}
context.safepoint();
}
return new BigIntegerStorage(out, n);
}
protected BigIntegerStorage runBigIntegerMap(
BigIntegerArrayAdapter a, BigInteger b, MapOperationProblemAggregator problemAggregator) {
if (b == null) {
return a.intoStorage();
}
Context context = Context.getCurrent();
int n = a.size();
BigInteger[] out = new BigInteger[n];
for (int i = 0; i < n; i++) {
BigInteger x = a.getItem(i);
if (x == null) {
out[i] = b;
} else {
BigInteger r = doBigInteger(x, b, i, problemAggregator);
out[i] = r;
}
context.safepoint();
}
return new BigIntegerStorage(out, n);
}
protected BigDecimalStorage runBigDecimalZip(
BigDecimalArrayAdapter a,
BigDecimalArrayAdapter b,
MapOperationProblemAggregator problemAggregator) {
Context context = Context.getCurrent();
int n = a.size();
int m = Math.min(a.size(), b.size());
BigDecimal[] out = new BigDecimal[n];
for (int i = 0; i < m; i++) {
BigDecimal x = a.getItem(i);
BigDecimal y = b.getItem(i);
if (x == null && y == null) {
out[i] = null;
} else {
if (x == null) {
out[i] = y;
} else if (y == null) {
out[i] = x;
} else {
BigDecimal r = doBigDecimal(x, y, i, problemAggregator);
out[i] = r;
}
}
context.safepoint();
}
return new BigDecimalStorage(out, n);
}
protected SpecializedStorage<BigDecimal> runBigDecimalMap(
BigDecimalArrayAdapter a, BigDecimal b, MapOperationProblemAggregator problemAggregator) {
if (b == null) {
return a.intoStorage();
}
Context context = Context.getCurrent();
int n = a.size();
BigDecimal[] out = new BigDecimal[n];
for (int i = 0; i < n; i++) {
BigDecimal x = a.getItem(i);
if (x == null) {
out[i] = b;
} else {
BigDecimal r = doBigDecimal(x, b, i, problemAggregator);
out[i] = r;
}
context.safepoint();
}
return new BigDecimalStorage(out, n);
}
}

5
std-bits/table/src/main/java/org/enso/table/data/column/operation/map/numeric/arithmetic/NumericBinaryOpImplementation.java
View File

@ -11,6 +11,7 @@ import org.enso.table.data.column.operation.map.MapOperationProblemAggregator;
import org.enso.table.data.column.operation.map.numeric.helpers.BigDecimalArrayAdapter;
import org.enso.table.data.column.operation.map.numeric.helpers.BigIntegerArrayAdapter;
import org.enso.table.data.column.operation.map.numeric.helpers.DoubleArrayAdapter;
import org.enso.table.data.column.storage.SpecializedStorage;
import org.enso.table.data.column.storage.Storage;
import org.enso.table.data.column.storage.numeric.AbstractLongStorage;
import org.enso.table.data.column.storage.numeric.BigDecimalStorage;
@ -247,7 +248,7 @@ public abstract class NumericBinaryOpImplementation<T extends Number, I extends
return new LongStorage(out, n, isNothing, INTEGER_RESULT_TYPE);
}
protected LongStorage runLongMap(
protected Storage<Long> runLongMap(
AbstractLongStorage a, Long b, MapOperationProblemAggregator problemAggregator) {
if (b == null) {
return LongStorage.makeEmpty(a.size(), INTEGER_RESULT_TYPE);
@ -338,7 +339,7 @@ public abstract class NumericBinaryOpImplementation<T extends Number, I extends
return new BigDecimalStorage(out, n);
}
protected BigDecimalStorage runBigDecimalMap(
protected SpecializedStorage<BigDecimal> runBigDecimalMap(
BigDecimalArrayAdapter a, BigDecimal b, MapOperationProblemAggregator problemAggregator) {
Context context = Context.getCurrent();
int n = a.size();

14
std-bits/table/src/main/java/org/enso/table/data/column/operation/map/numeric/helpers/BigDecimalArrayAdapter.java
View File

@ -13,6 +13,15 @@ public interface BigDecimalArrayAdapter {
int size();
default SpecializedStorage<BigDecimal> intoStorage() {
int n = size();
BigDecimal[] values = new BigDecimal[n];
for (int i = 0; i < n; i++) {
values[i] = getItem(i);
}
return new BigDecimalStorage(values, n);
}
static BigDecimalArrayAdapter fromStorage(SpecializedStorage<BigDecimal> storage) {
return new BigDecimalStorageAsBigDecimal(storage);
}
@ -56,6 +65,11 @@ public interface BigDecimalArrayAdapter {
public int size() {
return storage.size();
}
@Override
public SpecializedStorage<BigDecimal> intoStorage() {
return storage;
}
}
class BigIntegerStorageAsBigDecimal implements BigDecimalArrayAdapter {

14
std-bits/table/src/main/java/org/enso/table/data/column/operation/map/numeric/helpers/BigIntegerArrayAdapter.java
View File

@ -9,6 +9,15 @@ public interface BigIntegerArrayAdapter {
int size();
default BigIntegerStorage intoStorage() {
int n = size();
BigInteger[] values = new BigInteger[n];
for (int i = 0; i < n; i++) {
values[i] = getItem(i);
}
return new BigIntegerStorage(values, n);
}
static BigIntegerArrayAdapter fromStorage(BigIntegerStorage storage) {
return new BigIntegerStorageAsBigInteger(storage);
}
@ -33,6 +42,11 @@ public interface BigIntegerArrayAdapter {
public int size() {
return storage.size();
}
@Override
public BigIntegerStorage intoStorage() {
return storage;
}
}
class LongStorageAsBigInteger implements BigIntegerArrayAdapter {

15
std-bits/table/src/main/java/org/enso/table/data/column/operation/map/numeric/helpers/DoubleArrayAdapter.java
View File

@ -2,6 +2,7 @@ package org.enso.table.data.column.operation.map.numeric.helpers;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.BitSet;
import org.enso.table.data.column.storage.Storage;
import org.enso.table.data.column.storage.numeric.AbstractLongStorage;
import org.enso.table.data.column.storage.numeric.BigDecimalStorage;
@ -15,6 +16,20 @@ public interface DoubleArrayAdapter {
int size();
default DoubleStorage intoStorage() {
int n = size();
long[] values = new long[n];
BitSet isNothing = new BitSet();
for (int i = 0; i < n; i++) {
if (isNothing(i)) {
isNothing.set(i);
} else {
values[i] = Double.doubleToRawLongBits(getItemAsDouble(i));
}
}
return new DoubleStorage(values, n, isNothing);
}
static DoubleArrayAdapter fromStorage(BigIntegerStorage storage) {
return new BigIntegerStorageAsDouble(storage);
}

81
std-bits/table/src/main/java/org/enso/table/data/column/operation/map/text/CoalescingStringStringOp.java Normal file
View File

@ -0,0 +1,81 @@
package org.enso.table.data.column.operation.map.text;
import org.enso.table.data.column.operation.map.MapOperationProblemAggregator;
import org.enso.table.data.column.storage.SpecializedStorage;
import org.enso.table.data.column.storage.Storage;
import org.enso.table.data.column.storage.StringStorage;
import org.enso.table.data.column.storage.type.TextType;
import org.enso.table.error.UnexpectedTypeException;
import org.graalvm.polyglot.Context;
public abstract class CoalescingStringStringOp extends StringStringOp {
public CoalescingStringStringOp(String name) {
super(name);
}
@Override
public Storage<?> runBinaryMap(
SpecializedStorage<String> storage,
Object arg,
MapOperationProblemAggregator problemAggregator) {
int size = storage.size();
if (arg == null) {
return storage;
} else if (arg instanceof String argString) {
String[] newVals = new String[size];
Context context = Context.getCurrent();
for (int i = 0; i < size; i++) {
if (storage.isNothing(i)) {
newVals[i] = argString;
} else {
newVals[i] = doString(storage.getItem(i), argString);
}
context.safepoint();
}
TextType argumentType = TextType.preciseTypeForValue(argString);
TextType newType = computeResultType((TextType) storage.getType(), argumentType);
return new StringStorage(newVals, size, newType);
} else {
throw new UnexpectedTypeException("a Text");
}
}
@Override
public Storage<?> runZip(
SpecializedStorage<String> storage,
Storage<?> arg,
MapOperationProblemAggregator problemAggregator) {
if (arg instanceof StringStorage v) {
int size = storage.size();
String[] newVals = new String[size];
Context context = Context.getCurrent();
for (int i = 0; i < size; i++) {
String a = storage.getItem(i);
String b = v.getItem(i);
String r;
if (a == null && b == null) {
r = null;
} else {
if (a == null) {
r = b;
} else if (b == null) {
r = a;
} else {
r = doString(a, b);
}
}
newVals[i] = r;
context.safepoint();
}
TextType newType = computeResultType((TextType) storage.getType(), v.getType());
return new StringStorage(newVals, size, newType);
} else {
throw new UnexpectedTypeException("a Text column");
}
}
}

6
std-bits/table/src/main/java/org/enso/table/data/column/operation/map/text/StringBooleanOp.java
View File

@ -19,7 +19,7 @@ public abstract class StringBooleanOp
protected abstract boolean doString(String a, String b);
protected boolean doObject(String a, Object o) {
throw new UnexpectedTypeException("a Text");
throw new UnexpectedTypeException("a Text", o.toString());
}
@Override
@ -90,8 +90,8 @@ public abstract class StringBooleanOp
for (int i = 0; i < storage.size(); i++) {
if (!storage.isNothing(i) && i < arg.size() && !arg.isNothing(i)) {
Object x = arg.getItemBoxed(i);
if (x instanceof String) {
if (doString(storage.getItem(i), (String) x)) {
if (x instanceof String str) {
if (doString(storage.getItem(i), str)) {
newVals.set(i);
}
} else {

8
std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/IsEmptyOperation.java
View File

@ -4,6 +4,7 @@ import org.enso.table.data.column.builder.BoolBuilder;
import org.enso.table.data.column.operation.UnaryOperation;
import org.enso.table.data.column.operation.map.MapOperationProblemAggregator;
import org.enso.table.data.column.storage.ColumnStorage;
import org.enso.table.data.column.storage.type.AnyObjectType;
import org.enso.table.data.column.storage.type.TextType;
/* An operation that checks if a column's row values are empty.
@ -19,7 +20,9 @@ public class IsEmptyOperation extends AbstractUnaryBooleanOperation {
@Override
public boolean canApply(ColumnStorage storage) {
return storage.getType() instanceof TextType;
var type = storage.getType();
// We also allow this operation on Mixed type to facilitate `internal_is_empty` helper.
return type instanceof TextType || type instanceof AnyObjectType;
}
@Override
@ -31,8 +34,7 @@ public class IsEmptyOperation extends AbstractUnaryBooleanOperation {
if (value instanceof String s) {
builder.appendBoolean(s.isEmpty());
} else {
throw new IllegalArgumentException(
"Unsupported type: " + value.getClass() + " (expected text type).");
builder.appendBoolean(false);
}
}
}

9
std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/IsNaNOperation.java
View File

@ -10,6 +10,7 @@ import org.enso.table.data.column.storage.ColumnDoubleStorage;
import org.enso.table.data.column.storage.ColumnLongStorage;
import org.enso.table.data.column.storage.ColumnStorage;
import org.enso.table.data.column.storage.ColumnStorageWithNothingMap;
import org.enso.table.data.column.storage.type.AnyObjectType;
public class IsNaNOperation extends AbstractUnaryBooleanOperation {
public static final String NAME = "is_nan";
@ -21,7 +22,9 @@ public class IsNaNOperation extends AbstractUnaryBooleanOperation {
@Override
public boolean canApply(ColumnStorage storage) {
return storage.getType().isNumeric();
var type = storage.getType();
// We also allow this operation on Mixed type to facilitate `internal_is_nan` helper.
return type.isNumeric() || type instanceof AnyObjectType;
}
@Override
@ -67,9 +70,7 @@ public class IsNaNOperation extends AbstractUnaryBooleanOperation {
switch (value) {
case Double d -> builder.appendBoolean(Double.isNaN(d));
case Float f -> builder.appendBoolean(Float.isNaN(f));
case Number ignored -> builder.appendBoolean(false);
default -> throw new IllegalArgumentException(
"Unsupported type: " + value.getClass() + " (expected numeric type).");
default -> builder.appendBoolean(false);
}
}
}

6
std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/NotOperation.java
View File

@ -26,11 +26,7 @@ public class NotOperation extends AbstractUnaryBooleanOperation {
public ColumnStorage apply(
ColumnStorage storage, MapOperationProblemAggregator problemAggregator) {
if (storage instanceof BoolStorage boolStorage) {
return new BoolStorage(
boolStorage.getValues(),
boolStorage.getIsNothingMap(),
boolStorage.size(),
!boolStorage.isNegated());
return boolStorage.makeNegated();
}
var builder = createBuilder(storage, problemAggregator);

718
std-bits/table/src/main/java/org/enso/table/data/column/storage/BoolStorage.java
View File

@ -3,12 +3,14 @@ package org.enso.table.data.column.storage;
import java.util.BitSet;
import java.util.List;
import java.util.function.IntFunction;
import org.enso.base.CompareException;
import org.enso.base.polyglot.Polyglot_Utils;
import org.enso.table.data.column.builder.Builder;
import org.enso.table.data.column.operation.map.BinaryMapOperation;
import org.enso.table.data.column.operation.map.MapOperationProblemAggregator;
import org.enso.table.data.column.operation.map.MapOperationStorage;
import org.enso.table.data.column.operation.map.bool.BooleanIsInOp;
import org.enso.table.data.column.storage.type.AnyObjectType;
import org.enso.table.data.column.storage.type.BooleanType;
import org.enso.table.data.column.storage.type.StorageType;
import org.enso.table.data.mask.OrderMask;
@ -46,6 +48,10 @@ public final class BoolStorage extends Storage<Boolean>
return new BoolStorage(new BitSet(), new BitSet(), size, r);
}
public BoolStorage makeNegated() {
return new BoolStorage(values, isNothing, size, !negated);
}
@Override
public int size() {
return size;
@ -98,7 +104,7 @@ public final class BoolStorage extends Storage<Boolean>
* accordingly. If `arg` is true, new values are `values || isMissing` and if `arg` is false, new
* values are `values && (~isMissing)`.
*/
private Storage<?> fillMissingBoolean(boolean arg) {
private BoolStorage fillMissingBoolean(boolean arg) {
final var newValues = (BitSet) values.clone();
if (arg != negated) {
newValues.or(isNothing);
@ -232,203 +238,49 @@ public final class BoolStorage extends Storage<Boolean>
private static MapOperationStorage<Boolean, BoolStorage> buildOps() {
MapOperationStorage<Boolean, BoolStorage> ops = new MapOperationStorage<>();
ops.add(
new BinaryMapOperation<>(Maps.EQ) {
@Override
public BoolStorage runBinaryMap(
BoolStorage storage,
Object arg,
MapOperationProblemAggregator problemAggregator) {
if (arg == null) {
return BoolStorage.makeEmpty(storage.size);
} else if (arg instanceof Boolean v) {
if (v) {
return storage;
} else {
return new BoolStorage(
storage.values, storage.isNothing, storage.size, !storage.negated);
}
} else {
return new BoolStorage(new BitSet(), storage.isNothing, storage.size, false);
}
}
@Override
public BoolStorage runZip(
BoolStorage storage,
Storage<?> arg,
MapOperationProblemAggregator problemAggregator) {
Context context = Context.getCurrent();
BitSet out = new BitSet();
BitSet isNothing = new BitSet();
for (int i = 0; i < storage.size; i++) {
if (!storage.isNothing(i) && i < arg.size() && !arg.isNothing(i)) {
if (((Boolean) storage.getItem(i)).equals(arg.getItemBoxed(i))) {
out.set(i);
}
} else {
isNothing.set(i);
}
context.safepoint();
}
return new BoolStorage(out, isNothing, storage.size, false);
}
})
.add(
new BinaryMapOperation<>(Maps.AND) {
@Override
public BoolStorage runBinaryMap(
BoolStorage storage,
Object arg,
MapOperationProblemAggregator problemAggregator) {
if (arg == null) {
if (storage.negated) {
var newMissing = new BitSet(storage.size);
newMissing.flip(0, storage.size);
newMissing.xor(storage.values);
return new BoolStorage(storage.values, newMissing, storage.size, true);
} else {
var newMissing = storage.isNothing.get(0, storage.size);
newMissing.or(storage.values);
return new BoolStorage(new BitSet(), newMissing, storage.size, false);
}
} else if (arg instanceof Boolean v) {
return v
? storage
: new BoolStorage(new BitSet(), new BitSet(), storage.size, false);
} else {
throw new UnexpectedTypeException("a Boolean");
}
}
@Override
public BoolStorage runZip(
BoolStorage storage,
Storage<?> arg,
MapOperationProblemAggregator problemAggregator) {
if (!(arg instanceof BoolStorage v)) {
throw new UnexpectedColumnTypeException("Boolean");
}
BitSet out = v.values.get(0, storage.size);
boolean negated;
if (storage.negated && v.negated) {
out.or(storage.values);
negated = true;
} else if (storage.negated) {
out.andNot(storage.values);
negated = false;
} else if (v.negated) {
out.flip(0, storage.size);
out.and(storage.values);
negated = false;
} else {
out.and(storage.values);
negated = false;
}
BitSet isNothing = BitSets.makeDuplicate(storage.isNothing);
isNothing.or(v.isNothing);
int current = isNothing.nextSetBit(0);
while (current != -1) {
var value = negated != out.get(current);
if (!value
&& (storage.getItemBoxed(current) == Boolean.FALSE
|| v.getItemBoxed(current) == Boolean.FALSE)) {
isNothing.clear(current);
}
current = isNothing.nextSetBit(current + 1);
}
return new BoolStorage(out, isNothing, storage.size, negated);
}
})
.add(
new BinaryMapOperation<>(Maps.OR) {
@Override
public BoolStorage runBinaryMap(
BoolStorage storage,
Object arg,
MapOperationProblemAggregator problemAggregator) {
if (arg == null) {
if (storage.negated) {
var newMissing = storage.isNothing.get(0, storage.size);
newMissing.or(storage.values);
return new BoolStorage(new BitSet(), newMissing, storage.size, true);
} else {
var newMissing = new BitSet(storage.size);
newMissing.flip(0, storage.size);
newMissing.xor(storage.values);
return new BoolStorage(storage.values, newMissing, storage.size, false);
}
} else if (arg instanceof Boolean v) {
return v
? new BoolStorage(new BitSet(), new BitSet(), storage.size, true)
: storage;
} else {
throw new UnexpectedTypeException("a Boolean");
}
}
@Override
public BoolStorage runZip(
BoolStorage storage,
Storage<?> arg,
MapOperationProblemAggregator problemAggregator) {
if (!(arg instanceof BoolStorage v)) {
throw new UnexpectedColumnTypeException("Boolean");
}
BitSet out = v.values.get(0, storage.size);
boolean negated;
if (storage.negated && v.negated) {
out.and(storage.values);
negated = true;
} else if (storage.negated) {
out.flip(0, storage.size);
out.and(storage.values);
negated = true;
} else if (v.negated) {
out.flip(0, storage.size);
out.or(storage.values);
negated = false;
} else {
out.or(storage.values);
negated = false;
}
BitSet isNothing = BitSets.makeDuplicate(storage.isNothing);
isNothing.or(v.isNothing);
int current = isNothing.nextSetBit(0);
while (current != -1) {
var value = negated != out.get(current);
if (value
&& (storage.getItemBoxed(current) == Boolean.TRUE
|| v.getItemBoxed(current) == Boolean.TRUE)) {
isNothing.clear(current);
}
current = isNothing.nextSetBit(current + 1);
}
return new BoolStorage(out, isNothing, storage.size, negated);
}
})
.add(new BooleanIsInOp());
ops.add(new BoolEq())
.add(new BoolAnd())
.add(new BoolOr())
.add(new BooleanIsInOp())
.add(new BoolLess())
.add(new BoolLessOrEqual())
.add(new BoolGreaterOrEqual())
.add(new BoolGreater())
.add(new BoolMin())
.add(new BoolMax());
return ops;
}
/** Creates a mask that selects elements corresponding to true entries in the passed storage. */
public static BitSet toMask(BoolStorage storage) {
BitSet mask = new BitSet();
mask.or(storage.getValues());
if (storage.isNegated()) {
mask.flip(0, storage.size());
}
BitSet mask = storage.normalize();
mask.andNot(storage.getIsNothingMap());
return mask;
}
/**
* Returns a BitSet representation of the storage. It is the same as the values BitSet, but with
* an assumption that the negated flag is false.
*/
private BitSet normalize() {
BitSet set = new BitSet();
set.or(this.values);
if (this.negated) {
set.flip(0, this.size);
}
return set;
}
/** Acts like {@link #normalize} but also negates the bits. */
private BitSet negateNormalize() {
BitSet set = new BitSet();
set.or(this.values);
if (!this.negated) {
set.flip(0, this.size);
}
return set;
}
@Override
public BoolStorage slice(int offset, int limit) {
int newSize = Math.min(size - offset, limit);
@ -478,4 +330,492 @@ public final class BoolStorage extends Storage<Boolean>
}
return getItem(index);
}
private static class BoolEq extends BinaryMapOperation<Boolean, BoolStorage> {
public BoolEq() {
super(Maps.EQ);
}
@Override
public BoolStorage runBinaryMap(
BoolStorage storage, Object arg, MapOperationProblemAggregator problemAggregator) {
if (arg == null) {
return BoolStorage.makeEmpty(storage.size);
} else if (arg instanceof Boolean v) {
if (v) {
return storage;
} else {
return storage.makeNegated();
}
} else {
return new BoolStorage(new BitSet(), storage.isNothing, storage.size, false);
}
}
@Override
public BoolStorage runZip(
BoolStorage storage, Storage<?> arg, MapOperationProblemAggregator problemAggregator) {
Context context = Context.getCurrent();
BitSet out = new BitSet();
BitSet isNothing = new BitSet();
for (int i = 0; i < storage.size; i++) {
if (!storage.isNothing(i) && i < arg.size() && !arg.isNothing(i)) {
if (((Boolean) storage.getItem(i)).equals(arg.getItemBoxed(i))) {
out.set(i);
}
} else {
isNothing.set(i);
}
context.safepoint();
}
return new BoolStorage(out, isNothing, storage.size, false);
}
}
private static class BoolAnd extends BinaryMapOperation<Boolean, BoolStorage> {
public BoolAnd() {
super(Maps.AND);
}
@Override
public BoolStorage runBinaryMap(
BoolStorage storage, Object arg, MapOperationProblemAggregator problemAggregator) {
if (arg == null) {
if (storage.negated) {
var newMissing = new BitSet(storage.size);
newMissing.flip(0, storage.size);
newMissing.xor(storage.values);
return new BoolStorage(storage.values, newMissing, storage.size, true);
} else {
var newMissing = storage.isNothing.get(0, storage.size);
newMissing.or(storage.values);
return new BoolStorage(new BitSet(), newMissing, storage.size, false);
}
} else if (arg instanceof Boolean v) {
return v ? storage : new BoolStorage(new BitSet(), new BitSet(), storage.size, false);
} else {
throw new UnexpectedTypeException("a Boolean");
}
}
@Override
public BoolStorage runZip(
BoolStorage storage, Storage<?> arg, MapOperationProblemAggregator problemAggregator) {
if (!(arg instanceof BoolStorage v)) {
throw new UnexpectedColumnTypeException("Boolean");
}
BitSet out = v.values.get(0, storage.size);
boolean negated;
if (storage.negated && v.negated) {
out.or(storage.values);
negated = true;
} else if (storage.negated) {
out.andNot(storage.values);
negated = false;
} else if (v.negated) {
out.flip(0, storage.size);
out.and(storage.values);
negated = false;
} else {
out.and(storage.values);
negated = false;
}
BitSet isNothing = BitSets.makeDuplicate(storage.isNothing);
isNothing.or(v.isNothing);
if (storage.size > v.size) {
isNothing.set(v.size, storage.size);
}
int current = isNothing.nextSetBit(0);
while (current != -1) {
Boolean a = storage.getItemBoxed(current);
Boolean b = (current < v.size) ? v.getItemBoxed(current) : null;
if (a == Boolean.FALSE || b == Boolean.FALSE) {
isNothing.clear(current);
boolean falseValue = negated;
out.set(current, falseValue);
}
current = isNothing.nextSetBit(current + 1);
}
return new BoolStorage(out, isNothing, storage.size, negated);
}
}
private static class BoolOr extends BinaryMapOperation<Boolean, BoolStorage> {
public BoolOr() {
super(Maps.OR);
}
@Override
public BoolStorage runBinaryMap(
BoolStorage storage, Object arg, MapOperationProblemAggregator problemAggregator) {
if (arg == null) {
if (storage.negated) {
var newMissing = storage.isNothing.get(0, storage.size);
newMissing.or(storage.values);
return new BoolStorage(new BitSet(), newMissing, storage.size, true);
} else {
var newMissing = new BitSet(storage.size);
newMissing.flip(0, storage.size);
newMissing.xor(storage.values);
return new BoolStorage(storage.values, newMissing, storage.size, false);
}
} else if (arg instanceof Boolean v) {
return v ? new BoolStorage(new BitSet(), new BitSet(), storage.size, true) : storage;
} else {
throw new UnexpectedTypeException("a Boolean");
}
}
@Override
public BoolStorage runZip(
BoolStorage storage, Storage<?> arg, MapOperationProblemAggregator problemAggregator) {
if (!(arg instanceof BoolStorage v)) {
throw new UnexpectedColumnTypeException("Boolean");
}
BitSet out = v.values.get(0, storage.size);
boolean negated;
if (storage.negated && v.negated) {
out.and(storage.values);
negated = true;
} else if (storage.negated) {
out.flip(0, storage.size);
out.and(storage.values);
negated = true;
} else if (v.negated) {
out.flip(0, storage.size);
out.or(storage.values);
negated = false;
} else {
out.or(storage.values);
negated = false;
}
BitSet isNothing = BitSets.makeDuplicate(storage.isNothing);
isNothing.or(v.isNothing);
if (storage.size > v.size) {
isNothing.set(v.size, storage.size);
}
int current = isNothing.nextSetBit(0);
while (current != -1) {
Boolean a = storage.getItemBoxed(current);
Boolean b = (current < v.size) ? v.getItemBoxed(current) : null;
if (a == Boolean.TRUE || b == Boolean.TRUE) {
isNothing.clear(current);
boolean trueValue = !negated;
out.set(current, trueValue);
}
current = isNothing.nextSetBit(current + 1);
}
return new BoolStorage(out, isNothing, storage.size, negated);
}
}
private abstract static class BoolCompareOp extends BinaryMapOperation<Boolean, BoolStorage> {
public BoolCompareOp(String name) {
super(name);
}
protected abstract boolean doCompare(boolean a, boolean b);
@Override
public Storage<?> runZip(
BoolStorage storage, Storage<?> arg, MapOperationProblemAggregator problemAggregator) {
if (arg instanceof BoolStorage argBoolStorage) {
BitSet out = new BitSet();
BitSet isNothing = new BitSet();
int n = storage.size;
int m = Math.min(n, argBoolStorage.size);
Context context = Context.getCurrent();
for (int i = 0; i < m; i++) {
if (storage.isNothing(i) || argBoolStorage.isNothing(i)) {
isNothing.set(i);
} else {
boolean a = storage.getItem(i);
boolean b = argBoolStorage.getItem(i);
boolean r = doCompare(a, b);
out.set(i, r);
}
context.safepoint();
}
isNothing.set(m, n);
return new BoolStorage(out, isNothing, storage.size, false);
} else if (arg.getType() instanceof AnyObjectType) {
BitSet out = new BitSet();
BitSet isNothing = new BitSet();
int n = storage.size;
int m = Math.min(n, arg.size());
Context context = Context.getCurrent();
for (int i = 0; i < m; i++) {
if (storage.isNothing(i) || arg.isNothing(i)) {
isNothing.set(i);
} else {
boolean a = storage.getItem(i);
Object b = arg.getItemBoxed(i);
if (b instanceof Boolean bBool) {
boolean r = doCompare(a, bBool);
out.set(i, r);
} else {
assert b != null;
throw new CompareException(a, b);
}
}
context.safepoint();
}
isNothing.set(m, n);
return new BoolStorage(out, isNothing, storage.size, false);
} else {
throw new UnexpectedColumnTypeException("Boolean");
}
}
}
private static class BoolLess extends BoolCompareOp {
public BoolLess() {
super(Maps.LT);
}
@Override
public Storage<?> runBinaryMap(
BoolStorage storage, Object arg, MapOperationProblemAggregator problemAggregator) {
if (arg == null) {
return BoolStorage.makeEmpty(storage.size);
}
if (arg instanceof Boolean b) {
if (b) {
// false is smaller than true, so we want to negate
return new BoolStorage(storage.negateNormalize(), storage.isNothing, storage.size, false);
} else {
// nothing is strictly smaller than false
return new BoolStorage(new BitSet(), storage.isNothing, storage.size, false);
}
} else {
throw new UnexpectedTypeException("a Boolean", arg.toString());
}
}
@Override
protected boolean doCompare(boolean a, boolean b) {
return !a && b;
}
}
private static class BoolLessOrEqual extends BoolCompareOp {
public BoolLessOrEqual() {
super(Maps.LTE);
}
@Override
public Storage<?> runBinaryMap(
BoolStorage storage, Object arg, MapOperationProblemAggregator problemAggregator) {
if (arg == null) {
return BoolStorage.makeEmpty(storage.size);
}
if (arg instanceof Boolean b) {
if (b) {
// everything is <= true
return new BoolStorage(new BitSet(), storage.isNothing, storage.size, true);
} else {
// false is <= false
return new BoolStorage(storage.negateNormalize(), storage.isNothing, storage.size, false);
}
} else {
throw new UnexpectedTypeException("a Boolean", arg.toString());
}
}
@Override
protected boolean doCompare(boolean a, boolean b) {
return !a || b;
}
}
private static class BoolGreater extends BoolCompareOp {
public BoolGreater() {
super(Maps.GT);
}
@Override
public Storage<?> runBinaryMap(
BoolStorage storage, Object arg, MapOperationProblemAggregator problemAggregator) {
if (arg == null) {
return BoolStorage.makeEmpty(storage.size);
}
if (arg instanceof Boolean b) {
if (b) {
// nothing is strictly greater than true
return new BoolStorage(new BitSet(), storage.isNothing, storage.size, false);
} else {
// true is > false, so we just return as-is
return storage;
}
} else {
throw new UnexpectedTypeException("a Boolean", arg.toString());
}
}
@Override
protected boolean doCompare(boolean a, boolean b) {
return a && !b;
}
}
private static class BoolGreaterOrEqual extends BoolCompareOp {
public BoolGreaterOrEqual() {
super(Maps.GTE);
}
@Override
public Storage<?> runBinaryMap(
BoolStorage storage, Object arg, MapOperationProblemAggregator problemAggregator) {
if (arg == null) {
return BoolStorage.makeEmpty(storage.size);
}
if (arg instanceof Boolean b) {
if (b) {
// true is >= true
return storage;
} else {
// everything is >= false
return new BoolStorage(new BitSet(), storage.isNothing, storage.size, true);
}
} else {
throw new UnexpectedTypeException("a Boolean", arg.toString());
}
}
@Override
protected boolean doCompare(boolean a, boolean b) {
return a || !b;
}
}
private abstract static class BoolCoalescingOp extends BinaryMapOperation<Boolean, BoolStorage> {
public BoolCoalescingOp(String name) {
super(name);
}
protected abstract boolean doOperation(boolean a, boolean b);
@Override
public Storage<?> runZip(
BoolStorage storage, Storage<?> arg, MapOperationProblemAggregator problemAggregator) {
if (arg instanceof BoolStorage argBoolStorage) {
int n = storage.size;
int m = Math.min(n, argBoolStorage.size());
BitSet out = new BitSet();
BitSet isNothing = new BitSet();
Context context = Context.getCurrent();
for (int i = 0; i < m; i++) {
boolean isNothingA = storage.isNothing(i);
boolean isNothingB = argBoolStorage.isNothing(i);
if (isNothingA && isNothingB) {
isNothing.set(i);
} else {
if (isNothingA) {
out.set(i, argBoolStorage.getItem(i));
} else if (isNothingB) {
out.set(i, storage.getItem(i));
} else {
out.set(i, doOperation(storage.getItem(i), argBoolStorage.getItem(i)));
}
}
context.safepoint();
}
for (int i = m; i < n; i++) {
if (storage.isNothing(i)) {
isNothing.set(i);
} else {
out.set(i, storage.getItem(i));
}
context.safepoint();
}
return new BoolStorage(out, isNothing, storage.size, false);
} else {
throw new UnexpectedColumnTypeException("Boolean");
}
}
}
private static class BoolMin extends BoolCoalescingOp {
public BoolMin() {
super(Maps.MIN);
}
@Override
public BoolStorage runBinaryMap(
BoolStorage storage, Object arg, MapOperationProblemAggregator problemAggregator) {
if (arg == null) {
return storage;
}
if (arg instanceof Boolean b) {
if (b) {
// true is larger than false, so we want to keep values as is, and fill missing ones with
// true
return storage.fillMissingBoolean(true);
} else {
// false is smaller than everything:
return BoolStorage.makeConstant(storage.size, false);
}
} else {
throw new UnexpectedTypeException("a Boolean", arg.toString());
}
}
@Override
protected boolean doOperation(boolean a, boolean b) {
return a && b;
}
}
private static class BoolMax extends BoolCoalescingOp {
public BoolMax() {
super(Maps.MAX);
}
@Override
public BoolStorage runBinaryMap(
BoolStorage storage, Object arg, MapOperationProblemAggregator problemAggregator) {
if (arg == null) {
return storage;
}
if (arg instanceof Boolean b) {
if (b) {
// true is larger than everything:
return BoolStorage.makeConstant(storage.size, true);
} else {
// false is smaller than true, so we only fill null gaps with it
return storage.fillMissingBoolean(false);
}
} else {
throw new UnexpectedTypeException("a Boolean", arg.toString());
}
}
@Override
protected boolean doOperation(boolean a, boolean b) {
return a || b;
}
}
}

13
std-bits/table/src/main/java/org/enso/table/data/column/storage/SpecializedStorage.java
View File

@ -189,4 +189,17 @@ public abstract class SpecializedStorage<T> extends Storage<T> {
return storage.size();
}
}
/**
* Returns the specialized storage casted to my own type, if it is of the same type; or null
* otherwise.
*/
@SuppressWarnings("unchecked")
public SpecializedStorage<T> castIfSameType(SpecializedStorage<?> storage) {
if (storage.getType().equals(getType())) {
return (SpecializedStorage<T>) storage;
} else {
return null;
}
}
}

2
std-bits/table/src/main/java/org/enso/table/data/column/storage/Storage.java
View File

@ -101,6 +101,8 @@ public abstract class Storage<T> implements ColumnStorage {
public static final String CONTAINS = "contains";
public static final String LIKE = "like";
public static final String IS_IN = "is_in";
public static final String MIN = "min";
public static final String MAX = "max";
}
/* Specifies if the given binary operation has a vectorized implementation available for this storage.*/

73
std-bits/table/src/main/java/org/enso/table/data/column/storage/StringStorage.java
View File

@ -1,10 +1,12 @@
package org.enso.table.data.column.storage;
import java.util.BitSet;
import org.enso.base.CompareException;
import org.enso.base.Text_Utils;
import org.enso.table.data.column.operation.map.BinaryMapOperation;
import org.enso.table.data.column.operation.map.MapOperationProblemAggregator;
import org.enso.table.data.column.operation.map.MapOperationStorage;
import org.enso.table.data.column.operation.map.text.CoalescingStringStringOp;
import org.enso.table.data.column.operation.map.text.LikeOp;
import org.enso.table.data.column.operation.map.text.StringBooleanOp;
import org.enso.table.data.column.operation.map.text.StringIsInOp;
@ -124,6 +126,34 @@ public final class StringStorage extends SpecializedStorage<String> {
return Text_Utils.contains(a, b);
}
});
t.add(
new StringComparisonOp(Maps.LT) {
@Override
protected boolean doString(String a, String b) {
return Text_Utils.compare_normalized(a, b) < 0;
}
});
t.add(
new StringComparisonOp(Maps.LTE) {
@Override
protected boolean doString(String a, String b) {
return Text_Utils.compare_normalized(a, b) <= 0;
}
});
t.add(
new StringComparisonOp(Maps.GT) {
@Override
protected boolean doString(String a, String b) {
return Text_Utils.compare_normalized(a, b) > 0;
}
});
t.add(
new StringComparisonOp(Maps.GTE) {
@Override
protected boolean doString(String a, String b) {
return Text_Utils.compare_normalized(a, b) >= 0;
}
});
t.add(new LikeOp());
t.add(new StringIsInOp<>());
t.add(
@ -138,6 +168,38 @@ public final class StringStorage extends SpecializedStorage<String> {
return TextType.concatTypes(a, b);
}
});
t.add(
new CoalescingStringStringOp(Maps.MIN) {
@Override
protected String doString(String a, String b) {
if (Text_Utils.compare_normalized(a, b) < 0) {
return a;
} else {
return b;
}
}
@Override
protected TextType computeResultType(TextType a, TextType b) {
return TextType.maxType(a, b);
}
});
t.add(
new CoalescingStringStringOp(Maps.MAX) {
@Override
protected String doString(String a, String b) {
if (Text_Utils.compare_normalized(a, b) > 0) {
return a;
} else {
return b;
}
}
@Override
protected TextType computeResultType(TextType a, TextType b) {
return TextType.maxType(a, b);
}
});
return t;
}
@ -179,4 +241,15 @@ public final class StringStorage extends SpecializedStorage<String> {
return getType();
}
}
private abstract static class StringComparisonOp extends StringBooleanOp {
public StringComparisonOp(String name) {
super(name);
}
@Override
protected boolean doObject(String a, Object o) {
throw new CompareException(a, o);
}
}
}

81
std-bits/table/src/main/java/org/enso/table/data/column/storage/datetime/DateStorage.java
View File

@ -1,8 +1,13 @@
package org.enso.table.data.column.storage.datetime;
import java.time.LocalDate;
import org.enso.base.CompareException;
import org.enso.table.data.column.builder.Builder;
import org.enso.table.data.column.builder.DateBuilder;
import org.enso.table.data.column.operation.map.MapOperationStorage;
import org.enso.table.data.column.operation.map.datetime.DateTimeIsInOp;
import org.enso.table.data.column.operation.map.datetime.TimeLikeBinaryOpReturningBoolean;
import org.enso.table.data.column.operation.map.datetime.TimeLikeCoalescingOperation;
import org.enso.table.data.column.storage.ObjectStorage;
import org.enso.table.data.column.storage.SpecializedStorage;
import org.enso.table.data.column.storage.type.DateType;
@ -21,6 +26,70 @@ public final class DateStorage extends SpecializedStorage<LocalDate> {
MapOperationStorage<LocalDate, SpecializedStorage<LocalDate>> t =
ObjectStorage.buildObjectOps();
t.add(new DateTimeIsInOp<>(LocalDate.class));
t.add(
new TimeLikeBinaryOpReturningBoolean<>(Maps.EQ, LocalDate.class) {
@Override
protected boolean doOperation(LocalDate a, LocalDate b) {
return a.isEqual(b);
}
@Override
protected boolean doOther(LocalDate a, Object b) {
return false;
}
});
t.add(
new DateComparisonOp(Maps.LT) {
@Override
protected boolean doOperation(LocalDate a, LocalDate b) {
return a.compareTo(b) < 0;
}
});
t.add(
new DateComparisonOp(Maps.LTE) {
@Override
protected boolean doOperation(LocalDate a, LocalDate b) {
return a.compareTo(b) <= 0;
}
});
t.add(
new DateComparisonOp(Maps.GT) {
@Override
protected boolean doOperation(LocalDate a, LocalDate b) {
return a.compareTo(b) > 0;
}
});
t.add(
new DateComparisonOp(Maps.GTE) {
@Override
protected boolean doOperation(LocalDate a, LocalDate b) {
return a.compareTo(b) >= 0;
}
});
t.add(
new TimeLikeCoalescingOperation<>(Maps.MIN, LocalDate.class) {
@Override
protected Builder createOutputBuilder(int size) {
return new DateBuilder(size);
}
@Override
protected LocalDate doOperation(LocalDate a, LocalDate b) {
return a.compareTo(b) < 0 ? a : b;
}
});
t.add(
new TimeLikeCoalescingOperation<>(Maps.MAX, LocalDate.class) {
@Override
protected Builder createOutputBuilder(int size) {
return new DateBuilder(size);
}
@Override
protected LocalDate doOperation(LocalDate a, LocalDate b) {
return a.compareTo(b) > 0 ? a : b;
}
});
return t;
}
@ -38,4 +107,16 @@ public final class DateStorage extends SpecializedStorage<LocalDate> {
public StorageType getType() {
return DateType.INSTANCE;
}
private abstract static class DateComparisonOp
extends TimeLikeBinaryOpReturningBoolean<LocalDate> {
public DateComparisonOp(String name) {
super(name, LocalDate.class);
}
@Override
protected boolean doOther(LocalDate a, Object b) {
throw new CompareException(a, b);
}
}
}

80
std-bits/table/src/main/java/org/enso/table/data/column/storage/datetime/DateTimeStorage.java
View File

@ -2,11 +2,15 @@ package org.enso.table.data.column.storage.datetime;
import java.time.Duration;
import java.time.ZonedDateTime;
import org.enso.base.CompareException;
import org.enso.table.data.column.builder.Builder;
import org.enso.table.data.column.builder.DateTimeBuilder;
import org.enso.table.data.column.builder.ObjectBuilder;
import org.enso.table.data.column.operation.map.GenericBinaryObjectMapOperation;
import org.enso.table.data.column.operation.map.MapOperationStorage;
import org.enso.table.data.column.operation.map.datetime.DateTimeIsInOp;
import org.enso.table.data.column.operation.map.datetime.TimeLikeBinaryOpReturningBoolean;
import org.enso.table.data.column.operation.map.datetime.TimeLikeCoalescingOperation;
import org.enso.table.data.column.storage.ObjectStorage;
import org.enso.table.data.column.storage.SpecializedStorage;
import org.enso.table.data.column.storage.type.DateTimeType;
@ -25,6 +29,46 @@ public final class DateTimeStorage extends SpecializedStorage<ZonedDateTime> {
MapOperationStorage<ZonedDateTime, SpecializedStorage<ZonedDateTime>> t =
ObjectStorage.buildObjectOps();
t.add(new DateTimeIsInOp<>(ZonedDateTime.class));
t.add(
new TimeLikeBinaryOpReturningBoolean<>(Maps.EQ, ZonedDateTime.class) {
@Override
protected boolean doOperation(ZonedDateTime a, ZonedDateTime b) {
return a.isEqual(b);
}
@Override
protected boolean doOther(ZonedDateTime a, Object b) {
return false;
}
});
t.add(
new DateTimeComparisonOp(Maps.LT) {
@Override
protected boolean doOperation(ZonedDateTime a, ZonedDateTime b) {
return a.compareTo(b) < 0;
}
});
t.add(
new DateTimeComparisonOp(Maps.LTE) {
@Override
protected boolean doOperation(ZonedDateTime a, ZonedDateTime b) {
return a.compareTo(b) <= 0;
}
});
t.add(
new DateTimeComparisonOp(Maps.GT) {
@Override
protected boolean doOperation(ZonedDateTime a, ZonedDateTime b) {
return a.compareTo(b) > 0;
}
});
t.add(
new DateTimeComparisonOp(Maps.GTE) {
@Override
protected boolean doOperation(ZonedDateTime a, ZonedDateTime b) {
return a.compareTo(b) >= 0;
}
});
t.add(
new GenericBinaryObjectMapOperation<
ZonedDateTime, SpecializedStorage<ZonedDateTime>, Duration>(
@ -39,6 +83,30 @@ public final class DateTimeStorage extends SpecializedStorage<ZonedDateTime> {
return Duration.between(other, value);
}
});
t.add(
new TimeLikeCoalescingOperation<>(Maps.MIN, ZonedDateTime.class) {
@Override
protected Builder createOutputBuilder(int size) {
return new DateTimeBuilder(size);
}
@Override
protected ZonedDateTime doOperation(ZonedDateTime a, ZonedDateTime b) {
return a.compareTo(b) < 0 ? a : b;
}
});
t.add(
new TimeLikeCoalescingOperation<>(Maps.MAX, ZonedDateTime.class) {
@Override
protected Builder createOutputBuilder(int size) {
return new DateTimeBuilder(size);
}
@Override
protected ZonedDateTime doOperation(ZonedDateTime a, ZonedDateTime b) {
return a.compareTo(b) > 0 ? a : b;
}
});
return t;
}
@ -56,4 +124,16 @@ public final class DateTimeStorage extends SpecializedStorage<ZonedDateTime> {
public StorageType getType() {
return DateTimeType.INSTANCE;
}
private abstract static class DateTimeComparisonOp
extends TimeLikeBinaryOpReturningBoolean<ZonedDateTime> {
public DateTimeComparisonOp(String name) {
super(name, ZonedDateTime.class);
}
@Override
protected boolean doOther(ZonedDateTime a, Object b) {
throw new CompareException(a, b);
}
}
}

80
std-bits/table/src/main/java/org/enso/table/data/column/storage/datetime/TimeOfDayStorage.java
View File

@ -2,11 +2,15 @@ package org.enso.table.data.column.storage.datetime;
import java.time.Duration;
import java.time.LocalTime;
import org.enso.base.CompareException;
import org.enso.table.data.column.builder.Builder;
import org.enso.table.data.column.builder.ObjectBuilder;
import org.enso.table.data.column.builder.TimeOfDayBuilder;
import org.enso.table.data.column.operation.map.GenericBinaryObjectMapOperation;
import org.enso.table.data.column.operation.map.MapOperationStorage;
import org.enso.table.data.column.operation.map.datetime.DateTimeIsInOp;
import org.enso.table.data.column.operation.map.datetime.TimeLikeBinaryOpReturningBoolean;
import org.enso.table.data.column.operation.map.datetime.TimeLikeCoalescingOperation;
import org.enso.table.data.column.storage.ObjectStorage;
import org.enso.table.data.column.storage.SpecializedStorage;
import org.enso.table.data.column.storage.type.StorageType;
@ -25,6 +29,46 @@ public final class TimeOfDayStorage extends SpecializedStorage<LocalTime> {
MapOperationStorage<LocalTime, SpecializedStorage<LocalTime>> t =
ObjectStorage.buildObjectOps();
t.add(new DateTimeIsInOp<>(LocalTime.class));
t.add(
new TimeLikeBinaryOpReturningBoolean<>(Maps.EQ, LocalTime.class) {
@Override
protected boolean doOperation(LocalTime a, LocalTime b) {
return a.equals(b);
}
@Override
protected boolean doOther(LocalTime a, Object b) {
return false;
}
});
t.add(
new TimeOfDayComparisonOp(Maps.LT) {
@Override
protected boolean doOperation(LocalTime a, LocalTime b) {
return a.compareTo(b) < 0;
}
});
t.add(
new TimeOfDayComparisonOp(Maps.LTE) {
@Override
protected boolean doOperation(LocalTime a, LocalTime b) {
return a.compareTo(b) <= 0;
}
});
t.add(
new TimeOfDayComparisonOp(Maps.GT) {
@Override
protected boolean doOperation(LocalTime a, LocalTime b) {
return a.compareTo(b) > 0;
}
});
t.add(
new TimeOfDayComparisonOp(Maps.GTE) {
@Override
protected boolean doOperation(LocalTime a, LocalTime b) {
return a.compareTo(b) >= 0;
}
});
t.add(
new GenericBinaryObjectMapOperation<LocalTime, SpecializedStorage<LocalTime>, Duration>(
Maps.SUB, LocalTime.class, TimeOfDayStorage.class) {
@ -38,6 +82,30 @@ public final class TimeOfDayStorage extends SpecializedStorage<LocalTime> {
return Duration.between(other, value);
}
});
t.add(
new TimeLikeCoalescingOperation<>(Maps.MIN, LocalTime.class) {
@Override
protected Builder createOutputBuilder(int size) {
return new TimeOfDayBuilder(size);
}
@Override
protected LocalTime doOperation(LocalTime a, LocalTime b) {
return a.compareTo(b) < 0 ? a : b;
}
});
t.add(
new TimeLikeCoalescingOperation<>(Maps.MAX, LocalTime.class) {
@Override
protected Builder createOutputBuilder(int size) {
return new TimeOfDayBuilder(size);
}
@Override
protected LocalTime doOperation(LocalTime a, LocalTime b) {
return a.compareTo(b) > 0 ? a : b;
}
});
return t;
}
@ -55,4 +123,16 @@ public final class TimeOfDayStorage extends SpecializedStorage<LocalTime> {
public StorageType getType() {
return TimeOfDayType.INSTANCE;
}
private abstract static class TimeOfDayComparisonOp
extends TimeLikeBinaryOpReturningBoolean<LocalTime> {
public TimeOfDayComparisonOp(String name) {
super(name, LocalTime.class);
}
@Override
protected boolean doOther(LocalTime a, Object b) {
throw new CompareException(a, b);
}
}
}

4
std-bits/table/src/main/java/org/enso/table/data/column/storage/numeric/AbstractLongStorage.java
View File

@ -6,6 +6,8 @@ import org.enso.table.data.column.operation.map.MapOperationStorage;
import org.enso.table.data.column.operation.map.numeric.LongRoundOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.AddOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.DivideOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.MaxOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.MinOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.ModOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.MulOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.PowerOp;
@ -110,6 +112,8 @@ public abstract class AbstractLongStorage extends NumericStorage<Long>
.add(new ModOp<>())
.add(new PowerOp<>())
.add(new LongRoundOp(Maps.ROUND))
.add(new MinOp<>())
.add(new MaxOp<>())
.add(new LessComparison<>())
.add(new LessOrEqualComparison<>())
.add(new EqualsComparison<>())

4
std-bits/table/src/main/java/org/enso/table/data/column/storage/numeric/BigDecimalStorage.java
View File

@ -4,6 +4,8 @@ import java.math.BigDecimal;
import org.enso.table.data.column.operation.map.MapOperationStorage;
import org.enso.table.data.column.operation.map.numeric.arithmetic.AddOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.BigDecimalDivideOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.MaxOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.MinOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.ModOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.MulOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.PowerOp;
@ -40,6 +42,8 @@ public final class BigDecimalStorage extends SpecializedStorage<BigDecimal> {
.add(new BigDecimalDivideOp<>())
.add(new PowerOp<>())
.add(new ModOp<>())
.add(new MinOp<>())
.add(new MaxOp<>())
.add(new LessComparison<>())
.add(new LessOrEqualComparison<>())
.add(new EqualsComparison<>())

4
std-bits/table/src/main/java/org/enso/table/data/column/storage/numeric/BigIntegerStorage.java
View File

@ -5,6 +5,8 @@ import java.math.BigInteger;
import org.enso.table.data.column.operation.map.MapOperationStorage;
import org.enso.table.data.column.operation.map.numeric.arithmetic.AddOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.DivideOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.MaxOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.MinOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.ModOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.MulOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.PowerOp;
@ -39,6 +41,8 @@ public class BigIntegerStorage extends SpecializedStorage<BigInteger> {
.add(new DivideOp<>())
.add(new ModOp<>())
.add(new PowerOp<>())
.add(new MinOp<>())
.add(new MaxOp<>())
.add(new LessComparison<>())
.add(new LessOrEqualComparison<>())
.add(new EqualsComparison<>())

9
std-bits/table/src/main/java/org/enso/table/data/column/storage/numeric/DoubleStorage.java
View File

@ -9,6 +9,8 @@ import org.enso.table.data.column.operation.map.MapOperationStorage;
import org.enso.table.data.column.operation.map.numeric.DoubleRoundOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.AddOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.DivideOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.MaxOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.MinOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.ModOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.MulOp;
import org.enso.table.data.column.operation.map.numeric.arithmetic.PowerOp;
@ -101,6 +103,11 @@ public final class DoubleStorage extends NumericStorage<Double>
return Double.longBitsToDouble(data[i]);
}
@Override
public DoubleStorage intoStorage() {
return this;
}
@Override
public boolean isBinaryOpVectorized(String op) {
return ops.isSupportedBinary(op);
@ -282,6 +289,8 @@ public final class DoubleStorage extends NumericStorage<Double>
.add(new ModOp<>())
.add(new PowerOp<>())
.add(new DoubleRoundOp(Maps.ROUND))
.add(new MinOp<>())
.add(new MaxOp<>())
.add(new LessComparison<>())
.add(new LessOrEqualComparison<>())
.add(new EqualsComparison<>())

5
std-bits/table/src/main/java/org/enso/table/error/UnexpectedTypeException.java
View File

@ -19,6 +19,11 @@ public class UnexpectedTypeException extends RuntimeException {
this.expected = expected;
}
public UnexpectedTypeException(String expected, String got) {
super("Unexpected value type. Expected " + expected + ", but got " + got + ".");
this.expected = expected;
}
/**
* @return the expected type descriptor
*/

13
test/Benchmarks/src/Main.enso
View File

@ -18,10 +18,12 @@ import project.Sum
import project.Table.Add_Row_Number
import project.Table.Aggregate
import project.Table.Arithmetic
import project.Table.Column_Comparisons
import project.Table.Column_From_Vector
import project.Table.Cross_Tab
import project.Table.Csv
import project.Table.Join
import project.Table.Mixed_Operations
import project.Table.Sorting
import project.Table.Internal.Multi_Value_Key
import project.Text.Build
@ -62,12 +64,14 @@ all_benchmarks =
builder.append Aggregate.collect_benches
builder.append Arithmetic.collect_benches
builder.append Add_Row_Number.collect_benches
builder.append Column_Comparisons.collect_benches
builder.append Column_From_Vector.collect_benches
builder.append Cross_Tab.collect_benches
builder.append Csv.collect_benches
builder.append Join.collect_benches
builder.append Sorting.collect_benches
builder.append Multi_Value_Key.collect_benches
builder.append Mixed_Operations.collect_benches
# Text
builder.append Build.collect_benches
@ -101,17 +105,22 @@ all_benchmarks =
main filter=Nothing =
benchmarks = if filter.is_nothing then all_benchmarks else
no_nothing x = x.is_nothing.not
filter_regex = Regex.compile filter
is_matching name =
# We always check simple plain text filter, but also if the filter was a valid regex we check a regex match.
(name.contains filter) || (filter_regex.is_error.not && (filter_regex.match name . is_nothing . not))
filter_benchmarks (b:Bench) = case b of
Bench.All groups ->
vec = groups.map filter_benchmarks . filter no_nothing
if vec.is_empty then Nothing else
Bench.All vec
Bench.Group n opts specs -> if n.contains filter then b else
Bench.Group n opts specs -> if is_matching n then b else
vec = specs.map filter_benchmarks . filter no_nothing
if vec.is_empty then Nothing else
Bench.Group n opts vec
Bench.Spec n _ -> if n.contains filter then b else Nothing
Bench.Spec n _ ->
if is_matching n then b else Nothing
all_benchmarks.map filter_benchmarks . filter no_nothing

110
test/Benchmarks/src/Table/Column_Comparisons.enso Normal file
View File

@ -0,0 +1,110 @@
from Standard.Base import all
from Standard.Table import Table, Value_Type
from Standard.Test import Bench
options = Bench.options . set_warmup (Bench.phase_conf 2 3) . set_measure (Bench.phase_conf 5 1)
create_ints num_rows:Integer -> Table =
ints1 = Vector.new num_rows i-> i
ints2 = Vector.new num_rows i->
case i % 3 of
0 -> -i
1 -> i + 1
2 -> i % 100
t = Table.new [["ints1", ints1], ["ints2", ints2]]
Runtime.assert ((t.at "ints1" . value_type) == Value_Type.Integer)
Runtime.assert ((t.at "ints2" . value_type) == Value_Type.Integer)
t
create_dates num_rows:Integer -> Table =
base_date = Date.new 2020 01 01
dates1 = Vector.new num_rows i-> base_date.date_add i ..Day
dates2 = Vector.new num_rows i->
x = case i % 3 of
0 -> -i
1 -> i + 1
2 -> i % 100
base_date.date_add x ..Day
t = Table.new [["dates1", dates1], ["dates2", dates2]]
Runtime.assert ((t.at "dates1" . value_type) == Value_Type.Date)
Runtime.assert ((t.at "dates2" . value_type) == Value_Type.Date)
t
create_texts num_rows:Integer -> Table =
texts1 = Vector.new num_rows i->
case i % 4 of
0 -> "abc"
1 -> ""
2 -> "ąęłśćżź"
3 -> "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"
texts2 = Vector.new num_rows i->
case i % 4 of
0 -> "def"
1 -> "......"
2 -> "żżż🚂🚂"
3 -> "a"
t = Table.new [["texts1", texts1], ["texts2", texts2]]
Runtime.assert ((t.at "texts1" . value_type) == Value_Type.Char)
Runtime.assert ((t.at "texts2" . value_type) == Value_Type.Char)
t
create_bools num_rows:Integer -> Table =
bools1 = Vector.new num_rows i-> i % 2 == 0
bools2 = Vector.new num_rows i-> i % 3 == 0
t = Table.new [["bools1", bools1], ["bools2", bools2]]
Runtime.assert ((t.at "bools1" . value_type) == Value_Type.Boolean)
Runtime.assert ((t.at "bools2" . value_type) == Value_Type.Boolean)
t
type Data
private Value ~ints ~dates ~texts ~bools
create num_rows = Data.Value (create_ints num_rows) (create_dates num_rows) (create_texts num_rows) (create_bools num_rows)
collect_benches = Bench.build builder->
num_rows = 500 * 1000
data = Data.create num_rows
builder.group ("Column_Comparisons_" + num_rows.to_text) options group_builder->
group_builder.specify "integer_constant" <|
(data.ints.at "ints1") <= 5
group_builder.specify "integer_column" <|
(data.ints.at "ints1") <= (data.ints.at "ints2")
group_builder.specify "date_constant" <|
(data.dates.at "dates1") <= (Date.new 2021 01 01)
group_builder.specify "date_column" <|
(data.dates.at "dates1") <= (data.dates.at "dates2")
group_builder.specify "text_constant" <|
(data.texts.at "texts1") <= "def"
group_builder.specify "text_column" <|
(data.texts.at "texts1") <= (data.texts.at "texts2")
group_builder.specify "bool_column" <|
(data.bools.at "bools1") <= (data.bools.at "bools2")
builder.group ("Column_Minmax_" + num_rows.to_text) options group_builder->
group_builder.specify "integer_constant" <|
(data.ints.at "ints1") . min 5
group_builder.specify "integer_column" <|
(data.ints.at "ints1") . min (data.ints.at "ints2")
group_builder.specify "date_constant" <|
(data.dates.at "dates1") . min (Date.new 2021 01 01)
group_builder.specify "date_column" <|
(data.dates.at "dates1") . min (data.dates.at "dates2")
group_builder.specify "text_constant" <|
(data.texts.at "texts1") . min "def"
group_builder.specify "text_column" <|
(data.texts.at "texts1") . min (data.texts.at "texts2")
group_builder.specify "bool_column" <|
(data.bools.at "bools1") . min (data.bools.at "bools2")
main = collect_benches . run_main

6
test/Benchmarks/src/Table/Csv.enso
View File

@ -10,9 +10,9 @@ type Lazy_Data
private Write ~table:Table
collect_benches = Bench.build builder->
assert Examples.csv_2500_rows.exists "Expecting the file to exist at "+Examples.csv_2500_rows.path
write_data = Lazy_Data.Write (Examples.csv_2500_rows . read)
write_data = Lazy_Data.Write <|
assert Examples.csv_2500_rows.exists "Expecting the file to exist at "+Examples.csv_2500_rows.path
Examples.csv_2500_rows . read
builder.group ("Read_csv_file") options group_builder->
group_builder.specify "data_csv" <|

35
test/Benchmarks/src/Table/Mixed_Operations.enso Normal file
View File

@ -0,0 +1,35 @@
from Standard.Base import all
from Standard.Table import Table, Value_Type
from Standard.Test import Bench
options = Bench.options . set_warmup (Bench.phase_conf 2 3) . set_measure (Bench.phase_conf 5 1)
create_table : Table
create_table num_rows =
mixed_text = Vector.new num_rows i->
case i % 4 of
0 -> "txt"
1 -> ""
2 -> 42
3 -> Nothing
t = Table.new [["mixed text", mixed_text]]
Runtime.assert ((t.at "mixed text" . value_type) == Value_Type.Mixed)
t
type Data
private Value ~table
create num_rows = Data.Value (create_table num_rows)
collect_benches = Bench.build builder->
num_rows = 500 * 1000
data = Data.create num_rows
builder.group ("Mixed_Column_Operations_" + num_rows.to_text) options group_builder->
group_builder.specify "is_blank" <|
data.table.at "mixed text" . is_blank
main = collect_benches . run_main

54
test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso
View File

@ -19,7 +19,7 @@ from Standard.Test import all
import enso_dev.Base_Tests.Data.Round_Spec
import project.Common_Table_Operations.Main.Char_Max_Size_After_Substring_Behavior
from project.Common_Table_Operations.Util import run_default_backend, within_table, is_float_or_decimal
from project.Common_Table_Operations.Util import all
main filter=Nothing = run_default_backend add_specs filter
@ -52,6 +52,18 @@ type Min_Max_Data
c = t.at "c"
[a, b, c, t]
type Min_Max_Datetime_Data
Value ~table
setup table_builder = Min_Max_Datetime_Data.Value <|
dates1 = [Date.new 2021 1 1, Date.new 2021 1 2, Date.new 2021 1 3]
dates2 = [Date.new 2021 1 1, Date.new 2021 1 3, Date.new 2000 12 1]
times1 = [Time_Of_Day.new 1 2 3 4, Time_Of_Day.new 3 4 5 6, Time_Of_Day.new 5 6 7 8]
times2 = [Time_Of_Day.new 1 2 3 4, Time_Of_Day.new 1 1 1 1, Time_Of_Day.new 10 10 10 10]
date_times1 = [Date_Time.new 2021 1 1 1 2 3 4, Date_Time.new 2021 1 2 3 4 5 6, Date_Time.new 2021 1 3 5 6 7 8]
date_times2 = [Date_Time.new 2021 1 1 1 2 3 4, Date_Time.new 2021 1 3 1 2 3 4, Date_Time.new 2000 12 1 10 10 10 10]
table_builder [["ix", [1, 2, 3]], ["dates1", dates1], ["dates2", dates2], ["times1", times1], ["times2", times2], ["date_times1", date_times1], ["date_times2", date_times2]] . sort "ix"
type Literal_Data
Value ~data
@ -1374,6 +1386,46 @@ add_specs suite_builder setup =
c10.to_vector . should_equal [True, False, True]
c10.value_type.should_equal Value_Type.Boolean
group_builder.specify "should allow Date/Time columns" pending=pending_datetime <|
dt_data = Min_Max_Datetime_Data.setup setup.table_builder
c1 = dt_data.table.at "dates1" . min (Date.new 2021 1 2)
c1.to_vector . should_equal [Date.new 2021 1 1, Date.new 2021 1 2, Date.new 2021 1 2]
c1.value_type.should_equal Value_Type.Date
c2 = dt_data.table.at "dates1" . min (dt_data.table.at "dates2")
c2.to_vector . should_equal [Date.new 2021 1 1, Date.new 2021 1 2, Date.new 2000 12 1]
c3 = dt_data.table.at "dates1" . max (Date.new 2021 1 2)
c3.to_vector . should_equal [Date.new 2021 1 2, Date.new 2021 1 2, Date.new 2021 1 3]
c4 = dt_data.table.at "dates1" . max (dt_data.table.at "dates2")
c4.to_vector . should_equal [Date.new 2021 1 1, Date.new 2021 1 3, Date.new 2021 1 3]
c5 = dt_data.table.at "times1" . min (Time_Of_Day.new 2 2 2)
c5.to_vector . should_equal [Time_Of_Day.new 1 2 3 4, Time_Of_Day.new 2 2 2, Time_Of_Day.new 2 2 2]
c5.value_type.should_equal Value_Type.Time
c6 = dt_data.table.at "times1" . min (dt_data.table.at "times2")
c6.to_vector . should_equal [Time_Of_Day.new 1 2 3 4, Time_Of_Day.new 1 1 1 1, Time_Of_Day.new 5 6 7 8]
c7 = dt_data.table.at "times1" . max (Time_Of_Day.new 2 2 2)
c7.to_vector . should_equal [Time_Of_Day.new 2 2 2, Time_Of_Day.new 3 4 5 6, Time_Of_Day.new 5 6 7 8]
c8 = dt_data.table.at "times1" . max (dt_data.table.at "times2")
c8.to_vector . should_equal [Time_Of_Day.new 1 2 3 4, Time_Of_Day.new 3 4 5 6, Time_Of_Day.new 10 10 10 10]
c9 = dt_data.table.at "date_times1" . min (Date_Time.new 2021 1 2 2 2 2)
c9.to_vector . should_equal_tz_agnostic [Date_Time.new 2021 1 1 1 2 3 4, Date_Time.new 2021 1 2 2 2 2, Date_Time.new 2021 1 2 2 2 2]
c10 = dt_data.table.at "date_times1" . min (dt_data.table.at "date_times2")
c10.to_vector . should_equal_tz_agnostic [Date_Time.new 2021 1 1 1 2 3 4, Date_Time.new 2021 1 2 3 4 5 6, Date_Time.new 2000 12 1 10 10 10 10]
c11 = dt_data.table.at "date_times1" . max (Date_Time.new 2021 1 2 2 2 2)
c11.to_vector . should_equal_tz_agnostic [Date_Time.new 2021 1 2 2 2 2, Date_Time.new 2021 1 2 3 4 5 6, Date_Time.new 2021 1 3 5 6 7 8]
c12 = dt_data.table.at "date_times1" . max (dt_data.table.at "date_times2")
c12.to_vector . should_equal_tz_agnostic [Date_Time.new 2021 1 1 1 2 3 4, Date_Time.new 2021 1 3 1 2 3 4, Date_Time.new 2021 1 3 5 6 7 8]
group_builder.specify "should check types" <|
[(.min), (.max)].each op->
op data.a data.c . should_fail_with Invalid_Value_Type

97
test/Table_Tests/src/In_Memory/Bool_Spec.enso Normal file
View File

@ -0,0 +1,97 @@
from Standard.Base import all
from Standard.Table import Column
from Standard.Test import all
main filter=Nothing =
suite = Test.build suite_builder->
add_specs suite_builder
suite.run_with_filter filter
## We use some tricks to make boolean operations faster.
Because these tricks rely on non-trivial logic, there is also a slightly higher risk of mistakes, so we add a better test coverage.
add_specs suite_builder = suite_builder.group "[In-Memory] Boolean Column operations" group_builder->
# We check all combinations of boolean operations.
xs = [True, False, Nothing, True, False, Nothing, True, False, Nothing]
ys = [True, True, True, False, False, False, Nothing, Nothing, Nothing]
with_transformed_columns f =
[False, True].each x_negated->
[False, True].each y_negated->
[False, True].each y_shortened->
xs_vec = if x_negated then xs.map (x-> if x.is_nothing then Nothing else x.not) else xs
ys_vec_0 = if y_negated then ys.map (y-> if y.is_nothing then Nothing else y.not) else ys
ys_vec = if y_shortened then ys_vec_0.drop (..Last 3) else ys_vec_0
x_col_0 = Column.from_vector "X" xs_vec
x_col = if x_negated then x_col_0.not else x_col_0
y_col_0 = Column.from_vector "Y" ys_vec
y_col = if y_negated then y_col_0.not else y_col_0
x_desc = (if x_negated then ["negated X"] else [])
y_desc = (if y_negated then ["negated Y"] else []) + (if y_shortened then ["shortened Y"] else [])
clue = if x_desc.is_empty && y_desc.is_empty then "" else (x_desc+y_desc).join ", " " (" ") "
Test.with_clue clue <|
f x_col y_col
group_builder.specify "comparisons" <| with_transformed_columns x_col-> y_col->
comparisons = [["<", (<)], ["<=", (<=)], [">", (>)], [">=", (>=)], ["==", (==)], ["!=", (!=)]]
comparisons.each p->
op_name = p.first
op = p.second
enso_op x y = if x.is_nothing || y.is_nothing then Nothing else op x y
Test.with_clue "("+op_name+"): " <|
got_1 = op x_col y_col
expected_1 = xs.zip ys enso_op
got_1.to_vector . should_equal expected_1
[True, False, Nothing].each scalar-> Test.with_clue "([X] "+op_name+" "+scalar.to_text+"): " <|
got_2 = op x_col scalar
expected_2 = xs.map x-> enso_op x scalar
got_2.to_vector . should_equal expected_2
group_builder.specify "logical" <| with_transformed_columns x_col-> y_col->
# Or and And work like in Database - if one value is missing, the result is set if that value being set to something would not influence the result.
enso_or x y = case Pair.Value x y of
Pair.Value True _ -> True
Pair.Value _ True -> True
Pair.Value False False -> False
Pair.Value _ _ -> Nothing
enso_and x y = case Pair.Value x y of
Pair.Value False _ -> False
Pair.Value _ False -> False
Pair.Value True True -> True
Pair.Value _ _ -> Nothing
comparisons = [["||", (||), enso_or], ["&&", (&&), enso_and]]
comparisons.each p->
op_name = p.first
op = p.second
enso_op = p.at 2
Test.with_clue "("+op_name+"): " <|
got_1 = op x_col y_col
expected_1 = xs.zip ys enso_op
got_1.to_vector . should_equal expected_1
[True, False, Nothing].each scalar-> Test.with_clue "([X] "+op_name+" "+scalar.to_text+"): " <|
got_2 = op x_col scalar
expected_2 = xs.map x-> enso_op x scalar
got_2.to_vector . should_equal expected_2
group_builder.specify "min/max" <| with_transformed_columns x_col-> y_col->
enso_min x y = if x.is_nothing then y else if y.is_nothing then x else if x < y then x else y
enso_max x y = if x.is_nothing then y else if y.is_nothing then x else if x > y then x else y
comparisons = [["min", (.min), enso_min], ["max", (.max), enso_max]]
comparisons.each p->
op_name = p.first
op = p.second
enso_op = p.at 2
Test.with_clue "("+op_name+"): " <|
got_1 = op x_col y_col
expected_1 = xs.zip ys enso_op
got_1.to_vector . should_equal expected_1
[True, False, Nothing].each scalar-> Test.with_clue op_name+"([X], "+scalar.to_text+"): " <|
got_2 = op x_col scalar
expected_2 = xs.map x-> enso_op x scalar
got_2.to_vector . should_equal expected_2

2
test/Table_Tests/src/In_Memory/Main.enso
View File

@ -3,6 +3,7 @@ from Standard.Base import all
from Standard.Test import all
import project.In_Memory.Aggregate_Column_Spec
import project.In_Memory.Bool_Spec
import project.In_Memory.Builders_Spec
import project.In_Memory.Column_Spec
import project.In_Memory.Column_Format_Spec
@ -23,6 +24,7 @@ import project.In_Memory.Table_Running_Spec
add_specs suite_builder =
Aggregate_Column_Spec.add_specs suite_builder
Bool_Spec.add_specs suite_builder
Builders_Spec.add_specs suite_builder
Column_Format_Spec.add_specs suite_builder
Column_Spec.add_specs suite_builder