| Title: | Icechunk Transactional Array Repositories |
|---|---|
| Description: | Bindings to the 'Icechunk' Rust library: storage, repositories, branches, tags, snapshots, sessions, commits, conflict resolution, and virtual chunk references with explicit authorisation of the locations they point to. Sessions can be handed to other native code as bytes. Chunk references of any snapshot can be read as a table. |
| Authors: | Michael D. Sumner [aut, cre], Authors of the dependency Rust crates [ctb, cph] (see inst/AUTHORS) |
| Maintainer: | Michael D. Sumner <[email protected]> |
| License: | Apache License (>= 2) |
| Version: | 0.0.0.9000 |
| Built: | 2026-11-16 11:50:40 UTC |
| Source: | https://github.com/hypertidy/tabularberg |
The conformance check for a round trip: array descriptions must agree
(dims, shape, chunks, dtype, fill, codec, attributes, coordinate values)
and every array's global refs rows must agree on index, presence,
resolved location, offset, size and inline bytes. Source paths are
compared after each side's own remap for target.
ic_byterefs_compare(x, y, target = NULL)ic_byterefs_compare(x, y, target = NULL)
x, y
|
byterefs stores (see |
target |
remap target used to resolve paths on both sides. |
a data frame of checks: array, check, ok, detail.
ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) now <- ic_to_byterefs(ic_session_readonly(repo, branch = "main")) then <- ic_to_byterefs(ic_session_readonly(repo, tag = "v1")) cmp <- ic_byterefs_compare(then, now) cmp[!cmp$ok, ]ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) now <- ic_to_byterefs(ic_session_readonly(repo, branch = "main")) then <- ic_to_byterefs(ic_session_readonly(repo, tag = "v1")) cmp <- ic_byterefs_compare(then, now) cmp[!cmp$ok, ]
One container per scheme and host (http/https), bucket (s3, gcs), or
directory (file), as the locations resolve under target.
ic_byterefs_containers( br, target = NULL, region = Sys.getenv("AWS_REGION", "us-east-1") )ic_byterefs_containers( br, target = NULL, region = Sys.getenv("AWS_REGION", "us-east-1") )
br |
a byterefs store (see |
target |
remap target; default |
region |
S3 region for s3 containers. |
a list of ic_container() objects; the prefixes attribute
holds the URL prefixes, suitable for authorize.
ex <- ic_example_repo() br <- ic_to_byterefs(ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main")) ic_byterefs_containers(br)ex <- ic_example_repo() br <- ic_to_byterefs(ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main")) ic_byterefs_containers(br)
Reads a byterefs table, either the SQLite working store (store.sqlite,
or an open DBI connection) or the Parquet "flat" export (a directory
holding arrays.json, sources.parquet and refs_<array>.parquet).
byterefs/0.1 and its predecessor mdim-refs/0.2 are accepted.
ic_byterefs_read(x)ic_byterefs_read(x)
x |
path to a |
a byterefs object: a list with format_version, meta (named
list of strings), sources, arrays and remap data frames, and
refs(array), a function returning that array's global refs rows
(dim_0.., present, path, offset, size, info, data) joined
to the source's etag, mtime and source_size. Read from SQLite,
refs() queries the file lazily, so the connection stays open: call
$close() when done (functions given a file path close it themselves).
ex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") # write the store's references as a byterefs SQLite file, read it back if (requireNamespace("RSQLite", quietly = TRUE)) { db <- tempfile(fileext = ".sqlite") ic_to_byterefs(s, dsn = db) br <- ic_byterefs_read(db) print(br) print(br$arrays[, c("array", "kind", "shape", "chunks", "dtype")]) print(head(br$refs("from_bin"))) br$close() }ex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") # write the store's references as a byterefs SQLite file, read it back if (requireNamespace("RSQLite", quietly = TRUE)) { db <- tempfile(fileext = ".sqlite") ic_to_byterefs(s, dsn = db) br <- ic_byterefs_read(db) print(br) print(br$arrays[, c("array", "kind", "shape", "chunks", "dtype")]) print(head(br$refs("from_bin"))) br$close() }
ic_chunk_refs() returns one row per stored chunk of a snapshot (or of
one array): its Zarr key and where its bytes are. kind is "native"
(bytes in the repository, location is chunks/<id> relative to the
repository root), "virtual" (bytes in another object, location is its
URL) or "inline" (bytes in the manifest). Chunks never written are
absent (fill value). This is the table a reader needs to fetch chunks
itself, and to check a repository against the reference table it was
built from.
ic_chunk_refs(session, path = NULL, inline = FALSE) ic_nodes(session) ic_virtual_locations(session)ic_chunk_refs(session, path = NULL, inline = FALSE) ic_nodes(session) ic_virtual_locations(session)
session |
an |
path |
an array path ( |
inline |
include inline chunk bytes as a list column. |
a data frame: path, coord (comma-separated 0-based chunk index), key, kind, location, offset, length, etag, last_modified (and inline).
ex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") ic_nodes(s) ic_chunk_refs(s, "/from_bin")[, c("coord", "kind", "location", "offset", "length")] ic_chunk_refs(s, inline = TRUE)[, c("path", "coord", "kind", "length")] ic_virtual_locations(s)ex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") ic_nodes(s) ic_chunk_refs(s, "/from_bin")[, c("coord", "kind", "location", "offset", "length")] ic_chunk_refs(s, inline = TRUE)[, c("path", "coord", "kind", "length")] ic_virtual_locations(s)
Commit, amend, flush, discard, rebase
ic_commit(session, message, metadata = NULL, allow_empty = FALSE) ic_amend(session, message, metadata = NULL) ic_flush(session, message, metadata = NULL) ic_discard(session) ic_rebase( session, solver = c("detect", "basic"), on_chunk = c("ours", "theirs", "fail") ) ic_status(session)ic_commit(session, message, metadata = NULL, allow_empty = FALSE) ic_amend(session, message, metadata = NULL) ic_flush(session, message, metadata = NULL) ic_discard(session) ic_rebase( session, solver = c("detect", "basic"), on_chunk = c("ours", "theirs", "fail") ) ic_status(session)
session |
a writable |
message |
commit message. |
metadata |
a named list stored with the snapshot. |
allow_empty |
commit even with no changes. |
solver |
|
on_chunk |
for |
A commit that loses a race raises a condition of class
icechunk_conflict; call ic_rebase() and commit again.
new snapshot id (commit/amend/flush).
ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) s <- ic_session_writable(repo, "main") ic_set(s, "zarr.json", charToRaw('{"zarr_format":3,"node_type":"group","attributes":{"v":2}}')) ic_status(s) id <- ic_commit(s, "attributes, take one", metadata = list(author = "example")) ic_session_info(s)$read_only # a committed session is done # replace that commit instead of adding another s <- ic_session_writable(repo, "main") ic_set(s, "zarr.json", charToRaw('{"zarr_format":3,"node_type":"group","attributes":{"v":3}}')) ic_amend(s, "attributes, take two") ic_ancestry(repo, branch = "main")$message[1:2] # two writers: the second commit conflicts, rebase onto the new head a <- ic_session_writable(repo, "main") b <- ic_session_writable(repo, "main") ic_set(a, "small/c/0/0", writeBin(as.double(1:6), raw(), size = 8, endian = "little")) ic_set(b, "small/c/1/1", writeBin(as.double(7:12), raw(), size = 8, endian = "little")) ic_commit(a, "writer a") res <- tryCatch(ic_commit(b, "writer b"), icechunk_conflict = function(e) e) class(res) ic_rebase(b) ic_commit(b, "writer b, rebased")ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) s <- ic_session_writable(repo, "main") ic_set(s, "zarr.json", charToRaw('{"zarr_format":3,"node_type":"group","attributes":{"v":2}}')) ic_status(s) id <- ic_commit(s, "attributes, take one", metadata = list(author = "example")) ic_session_info(s)$read_only # a committed session is done # replace that commit instead of adding another s <- ic_session_writable(repo, "main") ic_set(s, "zarr.json", charToRaw('{"zarr_format":3,"node_type":"group","attributes":{"v":3}}')) ic_amend(s, "attributes, take two") ic_ancestry(repo, branch = "main")$message[1:2] # two writers: the second commit conflicts, rebase onto the new head a <- ic_session_writable(repo, "main") b <- ic_session_writable(repo, "main") ic_set(a, "small/c/0/0", writeBin(as.double(1:6), raw(), size = 8, endian = "little")) ic_set(b, "small/c/1/1", writeBin(as.double(7:12), raw(), size = 8, endian = "little")) ic_commit(a, "writer a") res <- tryCatch(ic_commit(b, "writer b"), icechunk_conflict = function(e) e) class(res) ic_rebase(b) ic_commit(b, "writer b, rebased")
Build information
ic_compiled_features() ic_icechunk_version()ic_compiled_features() ic_icechunk_version()
ic_compiled_features() the Cargo features in this build;
ic_icechunk_version() the icechunk crate version.
ic_compiled_features() ic_icechunk_version()ic_compiled_features() ic_icechunk_version()
Virtual chunk references point outside the repository. A repository declares containers (URL prefixes plus how to reach them), and the reader must authorise each prefix it is willing to read, with credentials. Nothing is authorised by default: this is the "where" control.
ic_cred_s3( kind = c("from_env", "anonymous", "static"), access_key_id = NULL, secret_access_key = NULL, session_token = NULL ) ic_cred_http() ic_cred_local() ic_cred_gcs(kind = c("from_env", "anonymous")) ic_cred_azure(kind = c("from_env", "anonymous")) ic_container( url_prefix, type = c("s3", "s3_compatible", "http", "local", "gcs", "in_memory"), region = NULL, endpoint_url = NULL, anonymous = FALSE, allow_http = FALSE, force_path_style = FALSE, path = NULL )ic_cred_s3( kind = c("from_env", "anonymous", "static"), access_key_id = NULL, secret_access_key = NULL, session_token = NULL ) ic_cred_http() ic_cred_local() ic_cred_gcs(kind = c("from_env", "anonymous")) ic_cred_azure(kind = c("from_env", "anonymous")) ic_container( url_prefix, type = c("s3", "s3_compatible", "http", "local", "gcs", "in_memory"), region = NULL, endpoint_url = NULL, anonymous = FALSE, allow_http = FALSE, force_path_style = FALSE, path = NULL )
kind |
for S3: |
access_key_id, secret_access_key, session_token
|
static S3 keys. |
url_prefix |
the prefix chunk locations must start with, e.g.
|
type |
store type for the container. |
region, endpoint_url, anonymous, allow_http, force_path_style
|
S3
options for |
path |
for |
a credential spec (a list), for authorize in ic_repo_open().
# a virtual chunk container: where chunk bytes may come from ic_container("https://noaa-cdr-sea-surface-temp-optimum-interpolation-pds.s3.amazonaws.com/", "http") ic_container("s3://my-bucket/data/", "s3", region = "us-west-2") ic_container("file:///data/netcdf/", "local", path = "/data/netcdf") # credentials for storage ic_cred_s3("anonymous") ## Not run: ic_cred_s3("static", access_key_id = "AKIA...", secret_access_key = "...") ## End(Not run)# a virtual chunk container: where chunk bytes may come from ic_container("https://noaa-cdr-sea-surface-temp-optimum-interpolation-pds.s3.amazonaws.com/", "http") ic_container("s3://my-bucket/data/", "s3", region = "us-west-2") ic_container("file:///data/netcdf/", "local", path = "/data/netcdf") # credentials for storage ic_cred_s3("anonymous") ## Not run: ic_cred_s3("static", access_key_id = "AKIA...", secret_access_key = "...") ## End(Not run)
Unpacks one of two Icechunk repositories shipped with the package: ten
days of NOAA OISST v2.1 daily sea surface temperature (September 1981),
from zarr-icechunk-assure.
The two are the same repository built twice, with the virtual references
written for NOAA's https:// archive or its s3:// bucket. Metadata,
coordinates and history read offline; chunk values are fetched from NOAA
when read, so they need the network. Each has two tags: as-virtualized
(the time array lacks its units attribute, a defect kept on purpose)
and cf-repaired (the units added by a metadata-only commit).
ic_example_oisst(cell = c("https", "s3"), path = tempfile("oisst-"))ic_example_oisst(cell = c("https", "s3"), path = tempfile("oisst-"))
cell |
|
path |
directory to unpack into (must not exist). |
A list with path (the repository), authorize (a named list:
the container's URL prefix and an anonymous credential, ready for
ic_repo_open()), probe (the contract from probe.json: shape,
codecs, prefixes and one probe pixel) and cell.
ic_example_repo() for a synthetic repository; the files are
in system.file("extdata", "oisst-sample", package = "tabularberg").
ox <- ic_example_oisst("https") repo <- ic_repo_open(ox$path) ic_tags(repo) # the repair between the two tags touched metadata only ic_diff(repo, from = list(tag = "as-virtualized"), to = list(tag = "cf-repaired")) # coordinates are stored in the repository; data chunks point at NOAA s <- ic_session_readonly(repo, tag = "cf-repaired") ic_chunk_refs(s, "/sst")[1:2, c("key", "location", "length")] # reading a data chunk needs the container authorised (and the network) s <- ic_session_readonly(ic_repo_open(ox$path, authorize = ox$authorize), tag = "cf-repaired") length(ic_get(s, "sst/c/0/0/0/0"))ox <- ic_example_oisst("https") repo <- ic_repo_open(ox$path) ic_tags(repo) # the repair between the two tags touched metadata only ic_diff(repo, from = list(tag = "as-virtualized"), to = list(tag = "cf-repaired")) # coordinates are stored in the repository; data chunks point at NOAA s <- ic_session_readonly(repo, tag = "cf-repaired") ic_chunk_refs(s, "/sst")[1:2, c("key", "location", "length")] # reading a data chunk needs the container authorised (and the network) s <- ic_session_readonly(ic_repo_open(ox$path, authorize = ox$authorize), tag = "cf-repaired") length(ic_get(s, "sst/c/0/0/0/0"))
Builds a self-contained Icechunk repository in a temporary directory, for examples and experiments. Nothing is downloaded. It holds:
ic_example_repo(path = tempfile("icechunk-example-"))ic_example_repo(path = tempfile("icechunk-example-"))
path |
directory to create (must not exist). |
/from_bin: float32, 64 x 48, chunks 16 x 48. Four virtual
references into a raw binary file grid.bin, one per 16-row block.
/from_nc: float32, 64 x 48, one chunk. A virtual reference into
grid.nc at offset 2048 (a 2048-byte header, then the same values).
/small: float64, 4 x 6, chunks 2 x 3, values 1:24 in C order,
written through the session in a second commit. Its chunks are small
enough (48 bytes) that Icechunk keeps them inline in the manifest.
The values of the virtual arrays are (0:3071) / 4, little-endian
float32 in C order. Both source files live in a directory next to the
repository, declared as a virtual chunk container. Like any container,
it must be authorised before its chunks are read: pass $authorize to
ic_repo_open() or DBI::dbConnect(). The first commit is tagged v1.
A list with path (the repository), authorize (the
container's URL prefix, file://...), source (the directory holding
grid.bin and grid.nc) and values (the virtual arrays' values).
ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) repo ic_nodes(ic_session_readonly(repo, branch = "main"))ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) repo ic_nodes(ic_session_readonly(repo, branch = "main"))
Reading an array's chunk references (ic_chunk_refs(),
ic_to_byterefs(), ic_refs_arrow(), the DBI refs_<array> tables)
needs every manifest that covers it. With split manifests on remote
storage, fetching them one after another dominates; tabularberg first
loads them into the session's cache this many at a time.
ic_manifest_concurrency(n = 32L)ic_manifest_concurrency(n = 32L)
n |
manifests fetched at once (1 = sequential). |
the previous value (invisibly).
old <- ic_manifest_concurrency(8L) old ic_manifest_concurrency(old)old <- ic_manifest_concurrency(8L) old ic_manifest_concurrency(old)
Returns an R6 store that pizzarr's pure-R code path accepts (it inherits
pizzarr::Store), so pizzarr without its Rust backend, as released on
CRAN, can read and write an Icechunk repository. Every key read or
written goes through the session: reads see the session's snapshot plus
its uncommitted changes, and writes stay in the session until
ic_commit().
ic_pizzarr_store(session)ic_pizzarr_store(session)
session |
an |
The store reports no store identifier, so pizzarr always uses its R
code path with it (for pizzarr's zarrs backend, use pizzarr's own
IcechunkStore, which takes the session as bytes).
Limits come from pizzarr's R decoder, not from Icechunk: Zarr v3 arrays
whose codecs it does not implement cannot be read this way
(sharding_indexed; and, in pizzarr 0.2.x as released, the netCDF-4
numcodecs.shuffle and numcodecs.zlib, which the mdsumner/pizzarr fork
adds). ic_get() still returns their raw chunk bytes.
an R6 object inheriting pizzarr::Store.
if (requireNamespace("pizzarr", quietly = TRUE) && requireNamespace("R6", quietly = TRUE)) { ex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") st <- ic_pizzarr_store(s) print(st) z <- pizzarr::zarr_open(st, mode = "r", path = "small") print(z$get_item("...")$data) # the virtual array, decoded from the source file's bytes v <- pizzarr::zarr_open(st, mode = "r", path = "from_nc")$get_item("...")$data all.equal(as.vector(t(v)), ex$values) }if (requireNamespace("pizzarr", quietly = TRUE) && requireNamespace("R6", quietly = TRUE)) { ex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") st <- ic_pizzarr_store(s) print(st) z <- pizzarr::zarr_open(st, mode = "r", path = "small") print(z$get_item("...")$data) # the virtual array, decoded from the source file's bytes v <- pizzarr::zarr_open(st, mode = "r", path = "from_nc")$get_item("...")$data all.equal(as.vector(t(v)), ex$values) }
Branches, tags, ancestry and diffs
ic_branches(repo) ic_tags(repo) ic_ancestry(repo, branch = NULL, tag = NULL, snapshot = NULL) ic_branch_create(repo, name, snapshot_id) ic_branch_reset(repo, name, snapshot_id, from = NULL) ic_branch_delete(repo, name) ic_tag_create(repo, name, snapshot_id) ic_tag_delete(repo, name) ic_lookup(repo, branch = "main", tag = NULL) ic_diff(repo, from, to)ic_branches(repo) ic_tags(repo) ic_ancestry(repo, branch = NULL, tag = NULL, snapshot = NULL) ic_branch_create(repo, name, snapshot_id) ic_branch_reset(repo, name, snapshot_id, from = NULL) ic_branch_delete(repo, name) ic_tag_create(repo, name, snapshot_id) ic_tag_delete(repo, name) ic_lookup(repo, branch = "main", tag = NULL) ic_diff(repo, from, to)
repo |
an |
branch, tag, snapshot
|
select one version (default branch "main"). |
name |
branch or tag name. |
snapshot_id |
a snapshot id. |
from |
for |
to |
for |
data frames for listings; snapshot ids as character.
ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) ic_branches(repo) ic_tags(repo) ic_ancestry(repo, branch = "main")[, c("snapshot_id", "message")] # a branch from the tagged snapshot, and what changed since v1 <- ic_lookup(repo, tag = "v1") ic_branch_create(repo, "dev", v1) ic_diff(repo, from = list(tag = "v1"), to = list(branch = "main")) ic_branch_delete(repo, "dev") # a real repository: a metadata repair recorded as a commit, no references rewritten oisst <- ic_repo_open(ic_example_oisst()$path) ic_ancestry(oisst, tag = "cf-repaired")[, c("snapshot_id", "message")] ic_diff(oisst, from = list(tag = "as-virtualized"), to = list(tag = "cf-repaired"))ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) ic_branches(repo) ic_tags(repo) ic_ancestry(repo, branch = "main")[, c("snapshot_id", "message")] # a branch from the tagged snapshot, and what changed since v1 <- ic_lookup(repo, tag = "v1") ic_branch_create(repo, "dev", v1) ic_diff(repo, from = list(tag = "v1"), to = list(branch = "main")) ic_branch_delete(repo, "dev") # a real repository: a metadata repair recorded as a commit, no references rewritten oisst <- ic_repo_open(ic_example_oisst()$path) ic_ancestry(oisst, tag = "cf-repaired")[, c("snapshot_id", "message")] ic_diff(oisst, from = list(tag = "as-virtualized"), to = list(tag = "cf-repaired"))
The byterefs refs view of an array (dim_0.., present, path,
offset, size, info, data), built in Rust as Arrow record batches
and handed over as a nanoarrow_array_stream, without R vectors in
between. path is dictionary-encoded (one entry per source), offset
and size are int64, as the byterefs spec has them. Paths have the
repository's remap rules undone (see ic_write_byterefs()); native
chunks (bytes held by the repository) have no path, and their bytes in
data when native = TRUE, as ic_to_byterefs() writes them.
ic_refs_arrow(session, path, chunks = NULL, batch_rows = 65536L, native = TRUE)ic_refs_arrow(session, path, chunks = NULL, batch_rows = 65536L, native = TRUE)
session |
an |
path |
array path. |
chunks |
optional slab, |
batch_rows |
rows per record batch. |
native |
fetch native chunks' bytes into |
The stream is lazy: it reads the session as it is when called (later writes to the session are not seen), and builds each batch as the consumer asks for it, with one batch of read-ahead, so a large array's refs never sit in memory whole. Errors met while streaming (a manifest that cannot be fetched) are raised by the consumer.
Consume with nanoarrow::convert_array_stream(), arrow::as_arrow_table(),
arrow::write_dataset() or duckdb::duckdb_register_arrow().
a nanoarrow_array_stream.
ex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") if (requireNamespace("nanoarrow", quietly = TRUE)) { st <- ic_refs_arrow(s, "from_bin", batch_rows = 2L) print(st$get_schema()) # batches are built as they are pulled while (!is.null(b <- st$get_next())) print(nanoarrow::convert_array(b)[, c("dim_0", "offset", "size")]) # one slab, with chunk bytes for inline/native chunks print(nanoarrow::convert_array_stream(ic_refs_arrow(s, "small", chunks = list(dim_0 = 1)))) }ex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") if (requireNamespace("nanoarrow", quietly = TRUE)) { st <- ic_refs_arrow(s, "from_bin", batch_rows = 2L) print(st$get_schema()) # batches are built as they are pulled while (!is.null(b <- st$get_next())) print(nanoarrow::convert_array(b)[, c("dim_0", "offset", "size")]) # one slab, with chunk bytes for inline/native chunks print(nanoarrow::convert_array_stream(ic_refs_arrow(s, "small", chunks = list(dim_0 = 1)))) }
Virtual chunk containers of a repository
ic_repo_containers(repo)ic_repo_containers(repo)
repo |
an |
a data frame: prefix (URL prefix of each declared container) and
authorized (whether this repository object may read from it).
ex <- ic_example_repo() repo <- ic_repo_open(ex$path) ic_repo_containers(repo) # declared, not yet authorised repo <- ic_repo_open(ex$path, authorize = "file://") ic_repo_containers(repo) # a broader prefix covers itex <- ic_example_repo() repo <- ic_repo_open(ex$path) ic_repo_containers(repo) # declared, not yet authorised repo <- ic_repo_open(ex$path, authorize = "file://") ic_repo_containers(repo) # a broader prefix covers it
Creates the repository with one virtual chunk container per source
prefix (ic_byterefs_containers()), authorises them, writes the store
with ic_write_byterefs() and commits.
ic_repo_from_byterefs( br, storage, target = NULL, message = NULL, authorize = NULL, manifest_split = NULL, ... )ic_repo_from_byterefs( br, storage, target = NULL, message = NULL, authorize = NULL, manifest_split = NULL, ... )
br |
a byterefs store (see |
storage |
an ic_storage object, or a local path, for the new repository. |
target |
remap target. |
message |
commit message. |
authorize |
credentials for the containers; default inferred from
each prefix's scheme (see |
manifest_split |
manifest splitting, see |
... |
passed to |
the icechunk_repo (invisibly), with attributes snapshot
and summary.
ex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") br <- ic_to_byterefs(s) # a new repository from the references: same chunks, same bytes dst <- tempfile("copy-") repo2 <- ic_repo_from_byterefs(br, dst, authorize = ex$authorize, message = "a copy") s2 <- ic_session_readonly(repo2, branch = "main") identical(ic_get(s2, "from_bin/c/2/0"), ic_get(s, "from_bin/c/2/0")) ic_byterefs_compare(br, ic_to_byterefs(s2)) # with one manifest per chunk row of dimension 0 dst3 <- tempfile("split-") ic_repo_from_byterefs(br, dst3, authorize = ex$authorize, manifest_split = c("0" = 1)) length(list.files(file.path(dst3, "manifests")))ex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") br <- ic_to_byterefs(s) # a new repository from the references: same chunks, same bytes dst <- tempfile("copy-") repo2 <- ic_repo_from_byterefs(br, dst, authorize = ex$authorize, message = "a copy") s2 <- ic_session_readonly(repo2, branch = "main") identical(ic_get(s2, "from_bin/c/2/0"), ic_get(s, "from_bin/c/2/0")) ic_byterefs_compare(br, ic_to_byterefs(s2)) # with one manifest per chunk row of dimension 0 dst3 <- tempfile("split-") ic_repo_from_byterefs(br, dst3, authorize = ex$authorize, manifest_split = c("0" = 1)) length(list.files(file.path(dst3, "manifests")))
Open or create an Icechunk repository
ic_repo_open(storage, containers = NULL, authorize = NULL) ic_repo_create( storage, containers = NULL, authorize = NULL, spec_version = NULL, manifest_split = NULL ) ic_repo_exists(storage) ic_repo_save_config(repo) ic_repo_config(repo)ic_repo_open(storage, containers = NULL, authorize = NULL) ic_repo_create( storage, containers = NULL, authorize = NULL, spec_version = NULL, manifest_split = NULL ) ic_repo_exists(storage) ic_repo_save_config(repo) ic_repo_config(repo)
storage |
an ic_storage object, or a local path. |
containers |
virtual chunk containers ( |
authorize |
virtual chunk prefixes this session may read: |
spec_version |
format version for new repositories (1 or 2;
|
manifest_split |
manifest splitting for new repositories: |
repo |
an |
an icechunk_repo.
# create a repository, write a group, commit path <- tempfile("ic-repo-open-") repo <- ic_repo_create(path) s <- ic_session_writable(repo, "main") ic_set(s, "zarr.json", charToRaw('{"zarr_format":3,"node_type":"group","attributes":{}}')) ic_commit(s, "an empty group") ic_repo_exists(path) # open it again; a repository with virtual chunk containers needs them # authorised before their chunks are read ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) repo ic_repo_config(repo)$virtual_chunk_containers[[1]]$url_prefix unlink(path, recursive = TRUE)# create a repository, write a group, commit path <- tempfile("ic-repo-open-") repo <- ic_repo_create(path) s <- ic_session_writable(repo, "main") ic_set(s, "zarr.json", charToRaw('{"zarr_format":3,"node_type":"group","attributes":{}}')) ic_commit(s, "an empty group") ic_repo_exists(path) # open it again; a repository with virtual chunk containers needs them # authorised before their chunks are read ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) repo ic_repo_config(repo)$virtual_chunk_containers[[1]]$url_prefix unlink(path, recursive = TRUE)
ic_session_bytes() serialises a session (Icechunk's own
Session::as_bytes) behind a small header: "ICR1", a u16
little-endian length, the icechunk crate version, then the payload. Any
consumer built against the same icechunk version can rebuild the session
(e.g. pizzarr's zarrs backend) and read or write at native speed.
ic_session_bytes(session) ic_session_from_bytes(bytes) ic_session_fork(session) ic_session_merge(session, bytes)ic_session_bytes(session) ic_session_from_bytes(bytes) ic_session_fork(session) ic_session_merge(session, bytes)
session |
an |
bytes |
a raw vector from |
For writes, hand out a ic_session_fork() of a writable session, take the
consumer's bytes back with ic_session_merge(), then ic_commit() the
base session. Commit authority stays here.
The bytes carry credentials. They hold the repository storage
(bucket, prefix, endpoint, and its credentials) and the virtual chunk
authorisations the repository was opened with. Static keys given to
ic_storage_s3() or to ic_cred_s3()("static", ...) are in the bytes in
plain text (verified for icechunk 2.3.1: S3ObjectStoreBackend and
S3Storage serialise their credentials field, and so does the virtual
chunk resolver). "from_env" and "anonymous" credentials carry no
secret: the consumer resolves them from its own environment. Prefer those
when bytes leave the process, and never log or persist bytes made with
static keys.
raw vector; a session; or nothing.
ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) s <- ic_session_writable(repo, "main") # hand a session to another process (or package) as bytes ... b <- ic_session_bytes(s) b s2 <- ic_session_from_bytes(b) ic_set(s2, "small/c/0/0", writeBin(as.double(6:1), raw(), size = 8, endian = "little")) # ... and merge its changes back before committing ic_session_merge(s, ic_session_bytes(s2)) ic_status(s) ic_commit(s, "written elsewhere")ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) s <- ic_session_writable(repo, "main") # hand a session to another process (or package) as bytes ... b <- ic_session_bytes(s) b s2 <- ic_session_from_bytes(b) ic_set(s2, "small/c/0/0", writeBin(as.double(6:1), raw(), size = 8, endian = "little")) # ... and merge its changes back before committing ic_session_merge(s, ic_session_bytes(s2)) ic_status(s) ic_commit(s, "written elsewhere")
A session is a view of one snapshot. Read-only sessions are pinned to a
branch tip, tag or snapshot id; writable sessions start from a branch tip
and accumulate changes until ic_commit().
ic_session_readonly(repo, branch = NULL, tag = NULL, snapshot = NULL) ic_session_writable(repo, branch = "main") ic_session_info(session)ic_session_readonly(repo, branch = NULL, tag = NULL, snapshot = NULL) ic_session_writable(repo, branch = "main") ic_session_info(session)
repo |
an |
branch, tag, snapshot
|
select one version (default branch "main"). |
session |
an |
an icechunk_session.
ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) # read the branch head, a tag, or a snapshot s <- ic_session_readonly(repo, branch = "main") ic_session_info(s) then <- ic_session_readonly(repo, tag = "v1") ic_nodes(then)$path # a writable session collects changes until ic_commit() w <- ic_session_writable(repo, "main") ic_set(w, "zarr.json", charToRaw('{"zarr_format":3,"node_type":"group","attributes":{"edited":true}}')) ic_session_info(w)$has_uncommitted_changes ic_discard(w) # one array's metadata at two tags of a real repository oisst <- ic_repo_open(ic_example_oisst()$path) units <- function(tag) { meta <- ic_get(ic_session_readonly(oisst, tag = tag), "time/zarr.json") jsonlite::fromJSON(rawToChar(meta))$attributes$units } units("as-virtualized") units("cf-repaired")ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) # read the branch head, a tag, or a snapshot s <- ic_session_readonly(repo, branch = "main") ic_session_info(s) then <- ic_session_readonly(repo, tag = "v1") ic_nodes(then)$path # a writable session collects changes until ic_commit() w <- ic_session_writable(repo, "main") ic_set(w, "zarr.json", charToRaw('{"zarr_format":3,"node_type":"group","attributes":{"edited":true}}')) ic_session_info(w)$has_uncommitted_changes ic_discard(w) # one array's metadata at two tags of a real repository oisst <- ic_repo_open(ic_example_oisst()$path) units <- function(tag) { meta <- ic_get(ic_session_readonly(oisst, tag = tag), "time/zarr.json") jsonlite::fromJSON(rawToChar(meta))$attributes$units } units("as-virtualized") units("cf-repaired")
Write virtual chunk references
ic_set_virtual_ref( session, key, location, offset, length, etag = NULL, last_modified = NULL, validate = TRUE ) ic_set_virtual_refs(session, path, refs, validate = TRUE)ic_set_virtual_ref( session, key, location, offset, length, etag = NULL, last_modified = NULL, validate = TRUE ) ic_set_virtual_refs(session, path, refs, validate = TRUE)
session |
a writable |
key |
Zarr chunk key. |
location |
URL of the object holding the bytes. |
offset, length
|
byte range in that object. |
etag, last_modified
|
optional checksum: reads fail if the object has changed. |
validate |
check the location falls within a declared container. |
path |
array path. |
refs |
data frame of 0-based chunk indices and byte ranges. Indices
are either |
number of refs that failed validation (invisibly).
ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) s <- ic_session_writable(repo, "main") grid <- paste0(ex$authorize, "grid.bin") # point from_bin's last chunk at the first block of grid.bin instead ic_set_virtual_ref(s, "from_bin/c/3/0", grid, offset = 0, length = 3072) # or many at once, one row per chunk (dim_* are chunk indices) ic_set_virtual_refs(s, "/from_bin", data.frame( dim_0 = 0:1, dim_1 = 0L, location = grid, offset = c(3072, 0), length = 3072)) ic_chunk_refs(s, "/from_bin")[, c("coord", "offset")] ic_commit(s, "rearranged from_bin")ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) s <- ic_session_writable(repo, "main") grid <- paste0(ex$authorize, "grid.bin") # point from_bin's last chunk at the first block of grid.bin instead ic_set_virtual_ref(s, "from_bin/c/3/0", grid, offset = 0, length = 3072) # or many at once, one row per chunk (dim_* are chunk indices) ic_set_virtual_refs(s, "/from_bin", data.frame( dim_0 = 0:1, dim_1 = 0L, location = grid, offset = c(3072, 0), length = 3072)) ic_chunk_refs(s, "/from_bin")[, c("coord", "offset")] ic_commit(s, "rearranged from_bin")
Where the repository lives. Storage objects hold no open connections and are cheap; credentials given here are never printed.
ic_storage_local(path) ic_storage_memory() ic_storage_http(base_url) ic_storage_s3( bucket, prefix = NULL, region = NULL, endpoint_url = NULL, anonymous = FALSE, allow_http = FALSE, force_path_style = !is.null(endpoint_url), requester_pays = FALSE, credentials = NULL, backend = c("object_store", "aws") ) ic_storage_gcs(bucket, prefix = NULL, anonymous = FALSE) ic_storage_azure(account, container, prefix = NULL)ic_storage_local(path) ic_storage_memory() ic_storage_http(base_url) ic_storage_s3( bucket, prefix = NULL, region = NULL, endpoint_url = NULL, anonymous = FALSE, allow_http = FALSE, force_path_style = !is.null(endpoint_url), requester_pays = FALSE, credentials = NULL, backend = c("object_store", "aws") ) ic_storage_gcs(bucket, prefix = NULL, anonymous = FALSE) ic_storage_azure(account, container, prefix = NULL)
path |
directory of the repository (local). |
base_url |
URL of the repository root (read-only HTTP). |
bucket, prefix
|
bucket and key prefix of the repository. |
region, endpoint_url
|
S3 region and endpoint (for S3-compatible services such as Pawsey Acacia or data.source.coop). |
anonymous |
unsigned requests. |
allow_http |
allow |
force_path_style |
path-style addressing (most S3-compatible services need this). |
requester_pays |
requester-pays bucket. |
credentials |
an |
backend |
|
account, container
|
Azure account and container. |
an icechunk_storage object.
st <- ic_storage_local(file.path(tempdir(), "a-repo")) st ic_storage_memory() ic_storage_http("https://data.source.coop/ausantarctic/ausantarctic-scratch/oisst/oisst-v2.1-daily/icechunk") ## Not run: # cloud object stores (the "cloud" build tier) ic_storage_s3("my-bucket", prefix = "repo", region = "us-west-2", credentials = ic_cred_s3("from_env")) ic_storage_s3("bucket", prefix = "repo", endpoint_url = "https://projects.pawsey.org.au", anonymous = TRUE) ic_storage_gcs("my-bucket", prefix = "repo", anonymous = TRUE) ic_storage_azure("account", "container", prefix = "repo") ## End(Not run)st <- ic_storage_local(file.path(tempdir(), "a-repo")) st ic_storage_memory() ic_storage_http("https://data.source.coop/ausantarctic/ausantarctic-scratch/oisst/oisst-v2.1-daily/icechunk") ## Not run: # cloud object stores (the "cloud" build tier) ic_storage_s3("my-bucket", prefix = "repo", region = "us-west-2", credentials = ic_cred_s3("from_env")) ic_storage_s3("bucket", prefix = "repo", endpoint_url = "https://projects.pawsey.org.au", anonymous = TRUE) ic_storage_gcs("my-bucket", prefix = "repo", anonymous = TRUE) ic_storage_azure("account", "container", prefix = "repo") ## End(Not run)
Raw bytes by Zarr key ("zarr.json", "temp/zarr.json",
"temp/c/0/0/0"). Decoding is a Zarr reader's job (pizzarr, GDAL).
ic_get(session, key, offset = NULL, length = NULL) ic_get_many(session, keys) ic_set(session, key, value) ic_exists(session, key) ic_delete(session, key) ic_list(session, prefix = "", recursive = TRUE)ic_get(session, key, offset = NULL, length = NULL) ic_get_many(session, keys) ic_set(session, key, value) ic_exists(session, key) ic_delete(session, key) ic_list(session, prefix = "", recursive = TRUE)
session |
an |
key, keys
|
Zarr keys. |
offset, length
|
optional byte range. |
value |
a raw vector. |
prefix |
key prefix; must be a group or array path (icechunk refuses
partial prefixes such as |
recursive |
list all keys below |
ic_get() a raw vector or NULL if absent (an absent chunk
means fill value); ic_get_many() a list of those.
ex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") ic_list(s, "small") ic_exists(s, "small/zarr.json") jsonlite::fromJSON(rawToChar(ic_get(s, "small/zarr.json")))$shape # a chunk's bytes: the virtual ones come from the source file x <- ic_get(s, "from_bin/c/1/0") head(readBin(x, "double", 3072, size = 4, endian = "little")) # a byte range of a chunk readBin(ic_get(s, "from_nc/c/0/0", offset = 0, length = 8), "double", 2, size = 4) lengths(ic_get_many(s, c("small/c/0/0", "small/c/1/1", "small/c/9/9"))) # writes go to a writable session w <- ic_session_writable(ic_repo_open(ex$path), "main") ic_set(w, "small/c/0/0", writeBin(as.double(101:106), raw(), size = 8, endian = "little")) ic_delete(w, "small/c/1/1") ic_status(w)ex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") ic_list(s, "small") ic_exists(s, "small/zarr.json") jsonlite::fromJSON(rawToChar(ic_get(s, "small/zarr.json")))$shape # a chunk's bytes: the virtual ones come from the source file x <- ic_get(s, "from_bin/c/1/0") head(readBin(x, "double", 3072, size = 4, endian = "little")) # a byte range of a chunk readBin(ic_get(s, "from_nc/c/0/0", offset = 0, length = 8), "double", 2, size = 4) lengths(ic_get_many(s, c("small/c/0/0", "small/c/1/1", "small/c/9/9"))) # writes go to a writable session w <- ic_session_writable(ic_repo_open(ex$path), "main") ic_set(w, "small/c/0/0", writeBin(as.double(101:106), raw(), size = 8, endian = "little")) ic_delete(w, "small/c/1/1") ic_status(w)
The inverse of ic_write_byterefs(), and the "chunk references as a
table" view of any Icechunk snapshot. Each array becomes an arrays row
(dtype, fill, codec and attributes from its zarr.json); virtual chunks
become refs rows with their source path, offset and size; inline and
native chunks become rows with a data blob (native chunk bytes are
fetched only if native = TRUE). Distinct locations become sources,
with etag / mtime from the chunk checksums.
ic_to_byterefs(session, dsn = NULL, native = TRUE)ic_to_byterefs(session, dsn = NULL, native = TRUE)
session |
an |
dsn |
optional path of a SQLite file to write (requires DBI and RSQLite); it must not exist. |
native |
fetch native chunk bytes into |
What Icechunk does not hold is reconstructed or left empty: source
size is NA, and placements are synthesised so the global refs view
is the identity (one placement per source, covering its chunks). If the
repository was written by ic_write_byterefs(), the root group's
byterefs entry restores the meta, the remap rules and the verbatim
source paths.
a byterefs object (invisibly if dsn is given).
ex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") br <- ic_to_byterefs(s) br$arrays[, c("array", "kind", "storage", "shape", "chunks")] br$refs("from_bin")[, c("dim_0", "dim_1", "path", "offset", "size")] br$sourcesex <- ic_example_repo() s <- ic_session_readonly(ic_repo_open(ex$path, authorize = ex$authorize), branch = "main") br <- ic_to_byterefs(s) br$arrays[, c("array", "kind", "storage", "shape", "chunks")] br$refs("from_bin")[, c("dim_0", "dim_1", "path", "offset", "size")] br$sources
Every present chunk becomes a virtual reference (location, offset,
length) to the source bytes, with the source's etag (or mtime) as
the checksum, so Icechunk refuses chunks whose source has changed.
(Icechunk checks "unmodified since"; for file:// locations the
recorded time is mtime + 1, because local filesystems keep sub-second
mtimes and byterefs records whole seconds.)
Chunks with an inline data blob are written as native chunks, absent
chunks are left unwritten (they read as the fill value), and coordinate
arrays (affine, inline) are materialised as single native chunks.
The root group's attributes hold meta, plus a byterefs entry with
the remap rules and target, so ic_to_byterefs() can recover the
verbatim source paths.
ic_write_byterefs( session, br, target = NULL, checksums = TRUE, validate = TRUE )ic_write_byterefs( session, br, target = NULL, checksums = TRUE, validate = TRUE )
session |
a writable |
br |
a byterefs store (see |
target |
remap target (see |
checksums |
record |
validate |
check every location falls within a declared container. |
Arrays become Zarr v3: dtype and byte order from dtype, bytes then
numcodecs.shuffle then zstd / numcodecs.zlib / gzip from codec,
with imagecodecs_delta (TIFF predictor 2) ahead of bytes.
a data frame, one row per array: array, refs (virtual),
native (inline or coordinate chunks), absent.
ex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) br <- ic_to_byterefs(ic_session_readonly(repo, branch = "main")) # edit the table, then write it into a session of another repository i <- which(br$arrays$array == "from_bin") br$arrays$attrs[[i]] <- list(units = "K") dst <- ic_repo_create(tempfile("dst-"), containers = ic_byterefs_containers(br), authorize = ex$authorize) w <- ic_session_writable(dst, "main") ic_write_byterefs(w, br) ic_commit(w, "from byterefs") jsonlite::fromJSON(rawToChar(ic_get(w, "from_bin/zarr.json")))$attributesex <- ic_example_repo() repo <- ic_repo_open(ex$path, authorize = ex$authorize) br <- ic_to_byterefs(ic_session_readonly(repo, branch = "main")) # edit the table, then write it into a session of another repository i <- which(br$arrays$array == "from_bin") br$arrays$attrs[[i]] <- list(units = "K") dst <- ic_repo_create(tempfile("dst-"), containers = ic_byterefs_containers(br), authorize = ex$authorize) w <- ic_session_writable(dst, "main") ic_write_byterefs(w, br) ic_commit(w, "from byterefs") jsonlite::fromJSON(rawToChar(ic_get(w, "from_bin/zarr.json")))$attributes
An Icechunk repository seen through DBI. Tables are the byterefs
projection of the snapshot the connection is pinned to (see
ic_to_byterefs()): meta, arrays, sources and one refs_<array>
per data array, plus Icechunk's own branches, tags, ancestry and
nodes. SQL (DBI::dbGetQuery()) runs on an in-memory SQLite copy of
the tables a statement names: an in-memory SQLite copy (RSQLite), or with
sql = "duckdb" a DuckDB registration that does not copy.
Icechunk() ## S4 method for signature 'IcechunkDriver' dbConnect( drv, storage, branch = NULL, tag = NULL, snapshot = NULL, authorize = NULL, containers = NULL, create = FALSE, target = NULL, sql = c("sqlite", "duckdb"), pushdown = TRUE, arrow = NA, ... ) ## S4 method for signature 'IcechunkConnection' dbListTables(conn, ...) ## S4 method for signature 'IcechunkConnection,character' dbExistsTable(conn, name, ...) ## S4 method for signature 'IcechunkConnection,character' dbReadTable(conn, name, ..., chunks = NULL) ## S4 method for signature 'IcechunkConnection,character' dbListFields(conn, name, ...) ## S4 method for signature 'IcechunkConnection,character' dbGetQuery(conn, statement, ...) ## S4 method for signature 'IcechunkConnection,character' dbSendQuery(conn, statement, ...) ## S4 method for signature 'IcechunkResult' dbFetch(res, n = -1, ...) ## S4 method for signature 'IcechunkResult' dbClearResult(res, ...) ## S4 method for signature 'IcechunkResult' dbHasCompleted(res, ...) ## S4 method for signature 'IcechunkResult' dbGetStatement(res, ...) ## S4 method for signature 'IcechunkResult' dbIsValid(dbObj, ...) ## S4 method for signature 'IcechunkConnection,character' dbExecute(conn, statement, ...) ## S4 method for signature 'IcechunkConnection' dbBegin(conn, ...) ## S4 method for signature 'IcechunkConnection' dbCommit( conn, message = "DBI commit", metadata = NULL, rebase = FALSE, solver = c("detect", "basic"), on_chunk = c("ours", "theirs", "fail"), tries = 5L, ... ) ## S4 method for signature 'IcechunkConnection' dbRollback(conn, ...) ## S4 method for signature 'IcechunkConnection' dbAppendTable(conn, name, value, ..., checksums = TRUE, row.names = NULL) ## S4 method for signature 'IcechunkConnection,character,data.frame' dbWriteTable(conn, name, value, ..., overwrite = FALSE, append = FALSE) ## S4 method for signature 'IcechunkConnection' dbDisconnect(conn, ...) ## S4 method for signature 'IcechunkConnection' dbIsValid(dbObj, ...) ## S4 method for signature 'IcechunkDriver' dbIsValid(dbObj, ...) ## S4 method for signature 'IcechunkConnection' dbGetInfo(dbObj, ...) ## S4 method for signature 'IcechunkDriver' dbGetInfo(dbObj, ...) ## S4 method for signature 'IcechunkConnection' show(object) ## S4 method for signature 'IcechunkDriver' show(object) ## S4 method for signature 'IcechunkDriver' dbDataType(dbObj, obj, ...)Icechunk() ## S4 method for signature 'IcechunkDriver' dbConnect( drv, storage, branch = NULL, tag = NULL, snapshot = NULL, authorize = NULL, containers = NULL, create = FALSE, target = NULL, sql = c("sqlite", "duckdb"), pushdown = TRUE, arrow = NA, ... ) ## S4 method for signature 'IcechunkConnection' dbListTables(conn, ...) ## S4 method for signature 'IcechunkConnection,character' dbExistsTable(conn, name, ...) ## S4 method for signature 'IcechunkConnection,character' dbReadTable(conn, name, ..., chunks = NULL) ## S4 method for signature 'IcechunkConnection,character' dbListFields(conn, name, ...) ## S4 method for signature 'IcechunkConnection,character' dbGetQuery(conn, statement, ...) ## S4 method for signature 'IcechunkConnection,character' dbSendQuery(conn, statement, ...) ## S4 method for signature 'IcechunkResult' dbFetch(res, n = -1, ...) ## S4 method for signature 'IcechunkResult' dbClearResult(res, ...) ## S4 method for signature 'IcechunkResult' dbHasCompleted(res, ...) ## S4 method for signature 'IcechunkResult' dbGetStatement(res, ...) ## S4 method for signature 'IcechunkResult' dbIsValid(dbObj, ...) ## S4 method for signature 'IcechunkConnection,character' dbExecute(conn, statement, ...) ## S4 method for signature 'IcechunkConnection' dbBegin(conn, ...) ## S4 method for signature 'IcechunkConnection' dbCommit( conn, message = "DBI commit", metadata = NULL, rebase = FALSE, solver = c("detect", "basic"), on_chunk = c("ours", "theirs", "fail"), tries = 5L, ... ) ## S4 method for signature 'IcechunkConnection' dbRollback(conn, ...) ## S4 method for signature 'IcechunkConnection' dbAppendTable(conn, name, value, ..., checksums = TRUE, row.names = NULL) ## S4 method for signature 'IcechunkConnection,character,data.frame' dbWriteTable(conn, name, value, ..., overwrite = FALSE, append = FALSE) ## S4 method for signature 'IcechunkConnection' dbDisconnect(conn, ...) ## S4 method for signature 'IcechunkConnection' dbIsValid(dbObj, ...) ## S4 method for signature 'IcechunkDriver' dbIsValid(dbObj, ...) ## S4 method for signature 'IcechunkConnection' dbGetInfo(dbObj, ...) ## S4 method for signature 'IcechunkDriver' dbGetInfo(dbObj, ...) ## S4 method for signature 'IcechunkConnection' show(object) ## S4 method for signature 'IcechunkDriver' show(object) ## S4 method for signature 'IcechunkDriver' dbDataType(dbObj, obj, ...)
drv |
an |
storage |
an ic_storage object or a local path. |
branch, tag, snapshot
|
what the connection is pinned to (one of them;
default branch |
authorize, containers
|
as for |
create |
create the repository if it does not exist. |
target |
remap target used to resolve |
sql |
SQL engine for |
pushdown |
for a statement with a plain conjunctive |
arrow |
with |
... |
unused. |
conn |
an |
name |
table name. |
chunks |
for |
statement |
SQL (read-only: SELECT over the tables it names). |
res |
an |
n |
rows to fetch (-1 for all). |
dbObj |
an Icechunk DBI object. |
message |
commit message. |
metadata |
commit metadata (named list). |
rebase |
if another writer moved the branch since |
solver, on_chunk
|
as for |
tries |
maximum commit attempts with |
value |
a data frame. For |
checksums |
record |
row.names |
ignored (required by the DBI generic). |
overwrite, append
|
only appending is supported. |
object |
an Icechunk DBI object. |
obj |
an R object. |
Writes go through a writable session on the connection's branch:
DBI::dbBegin() opens it, DBI::dbAppendTable() on refs_<array> sets
virtual (or inline, from data) chunk refs, on arrays creates arrays,
DBI::dbCommit() commits (pass message), DBI::dbRollback()
discards. Without dbBegin() each append commits on its own.
Icechunk(): an IcechunkDriver.
if (requireNamespace("RSQLite", quietly = TRUE)) { library(DBI) ex <- ic_example_repo() con <- dbConnect(Icechunk(), ex$path, authorize = ex$authorize) print(con) print(dbListTables(con)) print(dbGetQuery(con, 'SELECT "array", kind, shape, chunks, dtype FROM arrays')) # chunk references: one row per chunk; WHERE on dim_k reads only that slab print(dbGetQuery(con, 'SELECT dim_0, path, "offset", size FROM refs_from_bin WHERE dim_0 >= 2')) print(dbReadTable(con, "ancestry")[, c("snapshot_id", "message")]) # write: append references in a transaction, which becomes one commit dbBegin(con) dbAppendTable(con, "refs_from_bin", data.frame( dim_0 = 3L, dim_1 = 0L, path = file.path(ex$source, "grid.bin"), offset = 0, size = 3072)) dbCommit(con, "point the last block at the first") print(dbGetQuery(con, 'SELECT dim_0, "offset" FROM refs_from_bin')) # an earlier state, read-only old <- dbConnect(Icechunk(), ex$path, tag = "v1", authorize = ex$authorize) print(dbListTables(old)) dbDisconnect(old); dbDisconnect(con) }if (requireNamespace("RSQLite", quietly = TRUE)) { library(DBI) ex <- ic_example_repo() con <- dbConnect(Icechunk(), ex$path, authorize = ex$authorize) print(con) print(dbListTables(con)) print(dbGetQuery(con, 'SELECT "array", kind, shape, chunks, dtype FROM arrays')) # chunk references: one row per chunk; WHERE on dim_k reads only that slab print(dbGetQuery(con, 'SELECT dim_0, path, "offset", size FROM refs_from_bin WHERE dim_0 >= 2')) print(dbReadTable(con, "ancestry")[, c("snapshot_id", "message")]) # write: append references in a transaction, which becomes one commit dbBegin(con) dbAppendTable(con, "refs_from_bin", data.frame( dim_0 = 3L, dim_1 = 0L, path = file.path(ex$source, "grid.bin"), offset = 0, size = 3072)) dbCommit(con, "point the last block at the first") print(dbGetQuery(con, 'SELECT dim_0, "offset" FROM refs_from_bin')) # an earlier state, read-only old <- dbConnect(Icechunk(), ex$path, tag = "v1", authorize = ex$authorize) print(dbListTables(old)) dbDisconnect(old); dbDisconnect(con) }