Cookbook

import "github.com/Gaurav-Gosain/golars/expr"

qty := expr.Int("qty")
price := expr.Float("price")

out, _ := lazy.FromDataFrame(df).
    Filter(expr.All(qty.Gt(2), price.Lt(50))).
    WithColumns(
        price.MulCol(qty.CastFloat64()).As("total").Expr,
        qty.Between(2, 5).Alias("in_range"),
    ).
    Collect(ctx)

tagList := golars.Col("tags").Str().Split(",")
nums := golars.Col("scores").Str().Split(" ").
    List().Eval(golars.Element().Cast(dtype.Int64()))

out, _ := golars.Lazy(df).
    Select(
        golars.Col("user"),
        tagList.List().Len().Alias("n_tags"),
        tagList.List().Contains("go").Alias("has_go"),
        tagList.List().Unique(true).List().JoinWith("|", true).Alias("unique_tags"),
        nums.List().Sum().Alias("total"),
        nums.List().Sort(true, false).List().Head(2).Alias("top2"),
        nums.List().Eval(golars.Element().Mul(golars.Lit(int64(10)))).Alias("x10"),
    ).
    Collect(ctx)
// "go,db,go" / "3 9 4" gives n_tags=3, has_go=true, unique_tags="go|db",
// total=16, top2=[9, 4], x10=[30, 90, 40].

packed, _ := golars.Lazy(df).
    Select(golars.Col("user"), golars.Struct(golars.Col("x"), golars.Col("y")).Alias("point")).
    Collect(ctx)

flat, _ := golars.Lazy(packed).
    Select(
        golars.Col("point").Struct().Field("y").Alias("point_y"),
        // Unnest expands into one output column per field.
        golars.Col("point").Struct().RenameFields("px", "py").Struct().Unnest(),
    ).
    Collect(ctx)
// columns: point_y, px, py

named, _ := golars.Lazy(df).
    Select(
        golars.Col("x").Name().Prefix("raw_"),                       // raw_x
        golars.Col("y").Mul(golars.Lit(2.0)).Name().Suffix("_x2"),   // y_x2
        golars.Col("user").Name().Map(strings.ToUpper),              // USER
    ).
    Collect(ctx)

df, _ := golars.Lazy(raw).
    WithColumns(golars.Col("ts").Str().ToDatetime("%Y-%m-%d %H:%M")).
    WithColumns(
        golars.Col("ts").Dt().Weekday().Alias("weekday"),     // 1 = Monday
        golars.Col("ts").Dt().Truncate("1h").Alias("hour"),
        golars.Col("ts").Dt().Strftime("%a %H:%M").Alias("label"),
        // Mean over the trailing 2 hours of each row's timestamp.
        golars.Col("temp").RollingMeanBy(golars.Col("ts"), "2h").Alias("temp_2h"),
    ).
    Collect(ctx)

// Attach a zone to naive timestamps, then convert for display.
tokyo, _ := golars.Lazy(df).
    Select(golars.Col("ts").Dt().ReplaceTimeZone("UTC").
        Dt().ConvertTimeZone("Asia/Tokyo").Alias("tokyo")).
    Collect(ctx)

buckets, _ := golars.Lazy(df).
    GroupByDynamic("ts", golars.DynamicGroupOptions{
        Every:             "3h",
        IncludeBoundaries: true,
    }).
    Agg(
        golars.Col("temp").Mean().Alias("avg_temp"),
        golars.Col("temp").Count().Alias("n"),
    ).
    Collect(ctx)
// _lower_boundary      _upper_boundary      ts                   avg_temp  n
// 2024-03-01T06:00:00  2024-03-01T09:00:00  2024-03-01T06:00:00  18.75     2
// ...

at := func(hms string) time.Time {
    t, _ := time.Parse(time.DateTime, "2024-05-01 "+hms)
    return t
}
tt, _ := golars.FromTimes("time", []time.Time{at("09:30:05"), at("09:31:00"), at("09:36:00")}, nil, golars.Milliseconds, "")
tsym, _ := golars.FromString("symbol", []string{"AAPL", "MSFT", "AAPL"}, nil)
qty, _ := golars.FromInt64("qty", []int64{100, 50, 75}, nil)
trades, _ := golars.NewDataFrame(tt, tsym, qty)
defer trades.Release()

qt, _ := golars.FromTimes("time", []time.Time{at("09:30:00"), at("09:30:00"), at("09:33:00")}, nil, golars.Milliseconds, "")
qsym, _ := golars.FromString("symbol", []string{"AAPL", "MSFT", "AAPL"}, nil)
bid, _ := golars.FromFloat64("bid", []float64{189.1, 410.0, 189.4}, nil)
quotes, _ := golars.NewDataFrame(qt, qsym, bid)
defer quotes.Release()

// Latest quote for the same symbol, at most two minutes old.
out, _ := trades.JoinAsof(ctx, quotes, golars.AsofOptions{
    On:        "time",
    By:        []string{"symbol"},
    Strategy:  dataframe.AsofBackward,
    Tolerance: "2m",
})
// bid: 189.1, 410, null (the 09:33 AAPL quote is 3 minutes old)

// The lazy form takes the same options.
near, _ := golars.Lazy(trades).
    JoinAsof(golars.Lazy(quotes), golars.AsofOptions{
        On: "time", By: []string{"symbol"}, Strategy: dataframe.AsofNearest,
    }).
    Collect(ctx)

bins, _ := golars.Lazy(df).
    WithColumns(
        golars.Col("v").Cut([]float64{5, 10}, expr.CutOptions{}).Alias("cut"),
        golars.Col("v").Cut([]float64{5, 10},
            expr.CutOptions{Labels: []string{"low", "mid", "high"}}).Alias("label"),
        golars.Col("v").QCutN(2, expr.CutOptions{}).Alias("half"),
        golars.Col("v").QCut([]float64{0.25, 0.75},
            expr.CutOptions{Labels: []string{"q1", "iqr", "q4"}}).Alias("quartile"),
    ).
    Collect(ctx)
// v=[1, 5, 9, 12, 20]
// cut:      (-inf, 5]  (-inf, 5]  (5, 10]  (10, inf]  (10, inf]
// quartile: q1         q1         iqr      iqr        q4

zscore := func(s *series.Series) (*series.Series, error) {
    mean, err := s.Mean()
    if err != nil {
        return nil, err
    }
    std, err := s.Std()
    if err != nil {
        return nil, err
    }
    return s.ApplyFloat64(func(x float64) float64 { return (x - mean) / std })
}

z, _ := golars.Lazy(df).
    Select(golars.Col("v").Cast(dtype.Float64()).MapBatches(zscore).Alias("z")).
    Collect(ctx)

// Per value, with Go types checked at compile time:
lower, _ := golars.Lazy(df).
    Select(golars.MapElements(golars.Col("symbol"), strings.ToLower).Alias("sym")).
    Collect(ctx)

out, _ := df.Unnest(ctx, "payload")
// struct {x:i64, y:str} becomes two top-level cols `x` and `y`.

out, _ := df.Explode(ctx, "tags")
// [[a, b, c], [], NULL, [d]] produces 3 + 1 + 1 + 1 = 6 rows.

out, _ := df.Upsample(ctx, "ts", "1d")
// missing daily slots are inserted with nulls in every other column.

fmt.Println(lazy.ExplainTree(lf.Plan()))
// SORT [total desc]
// └── AGG keys=[dept] [col("salary").sum().alias("total")]
//     └── FILTER (col("salary") > 75)
//         └── SCAN df

df, _ := golars.ReadCSV("trades.csv")
defer df.Release()

out, _ := golars.Lazy(df).
    Filter(golars.Col("volume").GtLit(int64(100))).
    Collect(ctx)
defer out.Release()

golars.WriteParquet(out, "heavy_trades.parquet")

agg, _ := golars.Lazy(df).
    GroupBy("symbol").
    Agg(
        golars.Sum("qty"),
        golars.Mean("price").Alias("avg_price"),
        golars.Max("price").Alias("hi"),
    ).
    Sort("qty", true).
    Collect(ctx)

trades, _ := golars.ReadCSV("trades.csv")
defer trades.Release()
lookup, _ := golars.ReadParquet("symbols.parquet")
defer lookup.Release()

out, _ := golars.Lazy(trades).
    Join(golars.Lazy(lookup), []string{"symbol"}, golars.InnerJoin).
    Collect(ctx)

import iocsv "github.com/Gaurav-Gosain/golars/io/csv"

// iocsv.Scan returns a LazyFrame that opens the file only when
// Collect runs. Combined with Filter + Select, the optimiser pushes
// the projection down through the scan.
lf := iocsv.Scan("/tmp/huge.csv").
    Filter(golars.Col("region").EqLit("us")).
    Select(golars.Col("symbol"), golars.Col("price"))

for batch, err := range lf.IterBatches(ctx) {
    if err != nil { log.Fatal(err) }
    // stream-process each batch here
    batch.Release()
}

clean, _ := golars.Lazy(df).DropNulls("price", "qty").Collect(ctx)
filled, _ := golars.Lazy(df).FillNull(int64(0)).Collect(ctx)

mask, _ := df.AnyNullMask(ctx)
defer mask.Release()
// `mask` is a boolean Series you can plug back into Filter to flag
// bad rows without dropping them.

import "github.com/Gaurav-Gosain/golars/selector"

numericOnly, _ := df.SelectBy(selector.Numeric())
noTimes := df.DropBy(selector.EndsWith("_ts"))

// Combinators: intersect, union, minus.
usdCols, _ := df.SelectBy(selector.Intersect(
    selector.Float(),
    selector.StartsWith("price_usd"),
))

rec := df.ToArrow()          // arrow.RecordBatch
tbl := df.ToArrowTable()     // arrow.Table (multi-chunk)
roundtrip, _ := dataframe.FromArrowTable(tbl)

out, _ := df.Apply(func(s *series.Series) (*series.Series, error) {
    if s.Name() != "email" { return s.Clone(), nil }
    return s.Str().Before("@")
})

base := golars.Lazy(df).
    Filter(golars.Col("active").EqLit(true)).
    Cache()

// Two downstream pipelines share the same filtered base.
top, _ := base.Sort("score", true).Head(10).Collect(ctx)
flag, _ := base.Filter(golars.Col("score").LtLit(0.5)).Collect(ctx)

out, _ := golars.Lazy(df).
    Select(golars.When(golars.Col("age").Gt(golars.Lit(18))).
        Then(golars.Lit("adult")).
        Otherwise(golars.Lit("minor")).
        Alias("category")).
    Collect(ctx)

// Rolling sum/mean/min/max/std/var with a fixed window.
out, _ := golars.Lazy(df).
    Select(
        golars.Col("price").RollingMean(30, 1).Alias("ma30"),
        golars.Col("price").RollingStd(30, 5).Alias("vol30"),
    ).Collect(ctx)

// Boolean mask for regex hits.
s, _ := series.FromString("s", []string{"a1", "xx", "b22"}, nil)
mask, _ := s.Str().ContainsRegex(`\d+`)

// Extract first capture group.
ids, _ := emails.Str().Extract(`@([a-z.]+)$`, 1)

// Count matches per row.
counts, _ := tokens.Str().CountMatchesRegex(`\w+`)

// Mirror of polars' df.pivot(index="id", on="cat", values="v").
wide, _ := df.Pivot(ctx, []string{"id"}, "cat", "v", dataframe.PivotSum)

// Per-group total broadcast back to every row.
out, _ := golars.Lazy(df).
    Select(golars.Col("revenue").Sum().Over("region").Alias("region_total")).
    Collect(ctx)

// Per-group rank.
ranked, _ := golars.Lazy(df).
    Select(golars.Col("score").Rank("dense").Over("cohort").Alias("rank_in_cohort")).
    Collect(ctx)

// Replace every NaN with 0 in float columns (integer cols pass through).
filled, _ := golars.Lazy(df).FillNan(0).Collect(ctx)

// Carry the last non-null value forward through consecutive nulls.
// Pass limit=3 to stop after three consecutive fills; limit=0 means unlimited.
ff, _ := golars.Lazy(df).ForwardFill(0).Collect(ctx)
bf, _ := golars.Lazy(df).BackwardFill(0).Collect(ctx)

// Per-column variant via Expr:
out, _ := golars.Lazy(df).
    WithColumns(golars.Col("price").ForwardFill(0).Alias("price")).
    Collect(ctx)

// Transpose: each input column becomes a row; output value type is
// the promoted float64 (polars' object-dtype fallback is not yet
// supported, so non-numeric inputs error).
wide, _ := df.Transpose(ctx, "column", "row")

// Unpivot (melt): turn value columns into two long-form columns:
//   variable (the original column name) + value.
long, _ := df.Unpivot(ctx, []string{"id"}, nil /* default: all non-id */)

// Partition: one DataFrame per distinct key tuple; ordered by first
// appearance. Caller releases each partition.
parts, _ := df.PartitionBy(ctx, "region", "symbol")
for _, p := range parts {
    defer p.Release()
}

topSellers, _ := df.TopK(ctx, 10, "revenue")
worstLatency, _ := df.BottomK(ctx, 5, "p99_ms")

// Pipe keeps chained code flat:
out, _ := df.Pipe(func(d *golars.DataFrame) (*golars.DataFrame, error) {
    return d.Filter(ctx, mask)
})

sk, _ := col.Skew()                    // polars default (biased)
sku, _ := col.SkewUnbiased()           // scipy bias=False
kk, _ := col.Kurtosis()                // excess kurtosis

c, _ := a.PearsonCorr(b)               // Pearson r
cov, _ := a.Covariance(b, 1)           // ddof=1

corrMat, _ := df.Corr(ctx)             // k-by-k frame
covMat, _ := df.Cov(ctx, 1)

approx, _ := col.ApproxNUnique()       // HLL estimate

// Trig + hyperbolic family: atan2, cbrt, sinh/cosh/tanh, log1p, expm1,
// radians/degrees, arccos/arcsin/arctan, cot, arcsinh/arccosh/arctanh.
radians, _ := col.Radians()
y, _ := colY.Arctan2(colX)

picked, _ := golars.Lazy(df).
    Select(golars.Coalesce(golars.Col("primary"), golars.Col("fallback"))).
    Collect(ctx)

joined, _ := golars.Lazy(df).
    Select(golars.ConcatStr("-", golars.Col("sym"), golars.Col("year"))).
    Collect(ctx)

// Build Series out of thin air:
ids, _ := golars.Lazy(df).
    WithColumns(golars.IntRange(0, 100, 1).Alias("idx")).
    Collect(ctx)

sw, _ := golars.NewIPCStreamWriter(conn, firstBatch)
for batch := range batches {
    sw.Write(ctx, batch)
}
sw.Close()

// Consumer side (also polars/pyarrow/DuckDB-compatible):
sr, _ := golars.NewIPCStreamReader(conn)
defer sr.Close()
for batch, err := range sr.Iter(ctx) {
    if err != nil { log.Fatal(err) }
    process(batch)
    batch.Release()
}

// Append a column that sums three others on a per-row basis.
withTotal, _ := golars.Lazy(df).
    SumHorizontal("total", "q1", "q2", "q3").
    Collect(ctx)
defer withTotal.Release()

// Or compute the reduction directly as a standalone Series.
total, _ := golars.SumHorizontal(ctx, df, "q1", "q2", "q3")
defer total.Release()

sums, _ := df.SumAll(ctx)     // one row, one column per numeric input
means, _ := df.MeanAll(ctx)
counts, _ := df.CountAll(ctx) // counts non-nulls for every column
nulls, _ := df.NullCountAll(ctx)

lf := golars.ScanCSV("huge.csv").
    Filter(golars.Col("region").EqLit("us")).
    Select(golars.Col("symbol"), golars.Col("price"))

out, _ := lf.Collect(ctx)

r, _ := golars.Lazy(df).
    Select(
        golars.Col("score").Rank("dense").Alias("rank"),
        golars.Col("score").PctChange(1).Alias("delta"),
    ).
    Collect(ctx)

out, _ := df.Apply(func(s *series.Series) (*series.Series, error) {
    switch s.DType().String() {
    case "i64":
        return s.ApplyInt64(func(v int64) int64 { return v * 2 })
    case "str":
        return s.Str().Upper()
    }
    return s.Clone(), nil
})

load data/trades.csv
filter volume > 100
with_row_index row
cast price f64
fill_null 0
rename volume as qty
sum qty
write out.parquet