Standardize and append a dataset¶

Here, we’ll learn

how to standardize a less well curated dataset
how to append it to the growing versioned collection

import lamindb as ln
import bionty as bt

ln.track("ManDYgmftZ8C0003")

Let’s now consider a less-well curated dataset:

adata = ln.core.datasets.anndata_pbmc68k_reduced()
# we don't trust the cell type annotation in this dataset
adata.obs.rename(columns={"cell_type": "cell_type_untrusted"}, inplace=True)
adata

Create a curator:

curate = ln.Curator.from_anndata(
    adata,
    var_index=bt.Gene.symbol,
    categoricals={"cell_type_untrusted": bt.CellType.name},
    organism="human",
)

Standardize & validate genes ¶

Let’s convert Gene symbols to Ensembl ids via standardize(). Note that this is a non-unique mapping and the first match is kept because the keep parameter in .standardize() defaults to "first":

adata.var["ensembl_gene_id"] = bt.Gene.standardize(
    adata.var.index,
    field=bt.Gene.symbol,
    return_field=bt.Gene.ensembl_gene_id,
    organism="human",
)
# use ensembl_gene_id as the index
adata.var.index.name = "symbol"
adata.var = adata.var.reset_index().set_index("ensembl_gene_id")

# we only want to save data with validated genes
validated = bt.Gene.validate(adata.var.index, bt.Gene.ensembl_gene_id, mute=True)
adata_validated = adata[:, validated].copy()

Here, we’ll use .raw:

adata_validated.raw = adata.raw[:, validated].to_adata()
adata_validated.raw.var.index = adata_validated.var.index

curator = ln.Curator.from_anndata(
    adata_validated,
    var_index=bt.Gene.ensembl_gene_id,
    categoricals={"cell_type_untrusted": bt.CellType.name},
    organism="human",
)

curator.validate()

Standardize & validate cell types ¶

None of the cell type names are valid.

We’ll now search the public ontology and add the name found in the dataset as a synonym to the top match found in the public ontology.

bionty = bt.CellType.public()  # access the public ontology through bionty
name_mapping = {}
for invalid_name in adata_validated.obs["cell_type_untrusted"].unique():
    ontology_id = bionty.search(invalid_name).iloc[0].ontology_id  # top search hit through iloc[0]
    record = bt.CellType.from_source(ontology_id=ontology_id)
    name_mapping[invalid_name] = record.name  # map the original name to standardized name
    record.save()
    # record.add_synonym(name)  # optionally save the invalid name as synonym so that it becomes searchable
# print the mapping
print(name_mapping)

We can now standardize cell type names using the search-based mapper:

adata_validated.obs["cell_type_untrusted_original"] = adata_validated.obs["cell_type_untrusted"]  # copy the original annotations
adata_validated.obs["cell_type_untrusted"] = adata_validated.obs["cell_type_untrusted_original"].map(name_mapping)

Now, all cell types are validated:

curator.validate()

Register ¶

artifact = curator.save_artifact(description="10x reference adata")

artifact.view_lineage()

artifact.describe()

Artifact(uid='weUxELC0oEXmgLUW0000', is_latest=True, description='10x reference adata', suffix='.h5ad', type='dataset', size=855476, hash='3kPqtHuXea07jsEaQ0yYmQ', n_observations=70, _hash_type='md5', _accessor='AnnData', visibility=1, _key_is_virtual=True, created_at=2024-10-19 07:28:21 UTC)
  Provenance
    .storage = '/home/runner/work/lamin-usecases/lamin-usecases/docs/test-scrna'
    .transform = 'Standardize and append a dataset'
    .run = 2024-10-19 07:28:11 UTC
    .created_by = 'testuser1'
  Labels
    .cell_types = 'CD16-positive, CD56-dim natural killer cell, human', 'dendritic cell', 'B cell, CD19-positive', 'effector memory CD4-positive, alpha-beta T cell, terminally differentiated', 'cytotoxic T cell', 'CD8-positive, CD25-positive, alpha-beta regulatory T cell', 'CD14-positive, CD16-negative classical monocyte', 'CD38-positive naive B cell', 'CD38-high pre-BCR positive cell'
  Features
    'cell_type_untrusted' = 'B cell, CD19-positive', 'CD14-positive, CD16-negative classical monocyte', 'CD16-positive, CD56-dim natural killer cell, human', 'CD38-high pre-BCR positive cell', 'CD38-positive naive B cell', 'CD8-positive, CD25-positive, alpha-beta regulatory T cell', 'cytotoxic T cell', 'dendritic cell', 'effector memory CD4-positive, alpha-beta T cell, terminally differentiated'
  Feature sets
    'var' = 'HES4', 'TNFRSF4', 'SSU72', 'PARK7', 'RBP7', 'SRM', 'MAD2L2', 'AGTRAP', 'TNFRSF1B', 'EFHD2', 'NECAP2', 'HP1BP3', 'C1QA', 'C1QB', 'HNRNPR', 'GALE', 'STMN1', 'CD52', 'FGR', 'ATP5IF1'
    'obs' = 'cell_type_untrusted'

Re-curate¶

We review the dataset and find all annotations trustworthy up there being a 'CD38-positive naive B cell'.

Inspecting the name_mapping in detail tells us 'CD8+/CD45RA+ Naive Cytotoxic' was erroneously mapped on a B cell.

Let us correct this and create a 'cell_type' feature that we can now trust.

name_mapping['CD38-positive naive B cell'] = 'cytotoxic T cell'

adata_validated.obs["cell_type"] = adata_validated.obs["cell_type_untrusted_original"].map(name_mapping)
adata_validated.obs["cell_type"].unique()

['dendritic cell', 'B cell, CD19-positive', 'effector memory CD4-positive, alpha-beta T ce..., 'cytotoxic T cell', 'CD8-positive, CD25-positive, alpha-beta regul..., 'CD14-positive, CD16-negative classical monocyte', 'CD16-positive, CD56-dim natural killer cell, ..., 'CD38-positive naive B cell', 'CD38-high pre-BCR positive cell']
Categories (9, object): ['CD8-positive, CD25-positive, alpha-beta regul..., 'effector memory CD4-positive, alpha-beta T ce..., 'cytotoxic T cell', 'CD38-positive naive B cell', ..., 'B cell, CD19-positive', 'CD38-high pre-BCR positive cell', 'CD16-positive, CD56-dim natural killer cell, ..., 'dendritic cell']

artifact_trusted = ln.Curator.from_anndata(
    adata_validated,
    var_index=bt.Gene.ensembl_gene_id,
    categoricals={"cell_type": bt.CellType.name, "cell_type_untrusted": bt.CellType.name},
    organism="human",
).save_artifact(
    description="10x reference adata, trusted cell type annotation",
    revises=artifact,
)

• 4 non-validated values are not saved in Feature.name: ['cell_type_untrusted_original', 'percent_mito', 'louvain', 'n_genes']!
      → to lookup values, use lookup().columns
      → to save, run add_new_from_columns

• saving validated records of 'var_index'

• saving validated terms of 'cell_type'

• saving validated terms of 'cell_type_untrusted'

✓ var_index is validated against Gene.ensembl_gene_id

✓ cell_type is validated against CellType.name

✓ cell_type_untrusted is validated against CellType.name

!    4 unique terms (66.70%) are not validated for name: n_genes, percent_mito, louvain, cell_type_untrusted_original

artifact_trusted.describe()

Show code cell output Hide code cell output

Artifact(uid='weUxELC0oEXmgLUW0001', is_latest=True, description='10x reference adata, trusted cell type annotation', suffix='.h5ad', type='dataset', size=847300, hash='H2rLY3hcV18srupj0xhyCQ', n_observations=70, _hash_type='md5', _accessor='AnnData', visibility=1, _key_is_virtual=True, created_at=2024-10-19 07:28:23 UTC)
  Provenance
    .storage = '/home/runner/work/lamin-usecases/lamin-usecases/docs/test-scrna'
    .transform = 'Standardize and append a dataset'
    .run = 2024-10-19 07:28:11 UTC
    .created_by = 'testuser1'
  Labels
    .cell_types = 'CD16-positive, CD56-dim natural killer cell, human', 'dendritic cell', 'B cell, CD19-positive', 'effector memory CD4-positive, alpha-beta T cell, terminally differentiated', 'cytotoxic T cell', 'CD8-positive, CD25-positive, alpha-beta regulatory T cell', 'CD14-positive, CD16-negative classical monocyte', 'CD38-positive naive B cell', 'CD38-high pre-BCR positive cell'
  Features
    'cell_type' = 'B cell, CD19-positive', 'CD14-positive, CD16-negative classical monocyte', 'CD16-positive, CD56-dim natural killer cell, human', 'CD38-high pre-BCR positive cell', 'CD38-positive naive B cell', 'CD8-positive, CD25-positive, alpha-beta regulatory T cell', 'cytotoxic T cell', 'dendritic cell', 'effector memory CD4-positive, alpha-beta T cell, terminally differentiated'
    'cell_type_untrusted' = 'B cell, CD19-positive', 'CD14-positive, CD16-negative classical monocyte', 'CD16-positive, CD56-dim natural killer cell, human', 'CD38-high pre-BCR positive cell', 'CD38-positive naive B cell', 'CD8-positive, CD25-positive, alpha-beta regulatory T cell', 'cytotoxic T cell', 'dendritic cell', 'effector memory CD4-positive, alpha-beta T cell, terminally differentiated'
  Feature sets
    'var' = 'HES4', 'TNFRSF4', 'SSU72', 'PARK7', 'RBP7', 'SRM', 'MAD2L2', 'AGTRAP', 'TNFRSF1B', 'EFHD2', 'NECAP2', 'HP1BP3', 'C1QA', 'C1QB', 'HNRNPR', 'GALE', 'STMN1', 'CD52', 'FGR', 'ATP5IF1'
    'obs' = 'cell_type', 'cell_type_untrusted'

Append the dataset to the collection¶

Query the previous collection:

collection_v1 = ln.Collection.get(name="My versioned scRNA-seq collection")

Create a new version of the collection by sharding it across the new artifact and the artifact underlying version 1 of the collection:

collection_v2 = collection_v1.append(artifact_trusted).save()

If you want, you can label the collection’s version by setting .version.

collection_v2.version = "2"
collection_v2.save()

Version 2 of the collection covers significantly more conditions.

collection_v2.describe()

View data lineage:

collection_v2.view_lineage()