{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "77e06288", "metadata": {}, "outputs": [], "source": [ "# imports\n", "import pathlib\n", "import random\n", "import concurrent.futures\n", "import tifffile\n", "import napari\n", "import zarr\n", "import numpy as np\n", "import dask.array as da\n", "import cv2\n", "from numcodecs import Blosc\n", "from OpenGL.GL import glGetIntegerv, GL_MAX_TEXTURE_SIZE\n", "from coda_visual.datasets.coda_dataset import CODADataset\n", "from coda_visual.datasets.coda_raw_file import CODARawFile" ] }, { "cell_type": "code", "execution_count": 2, "id": "f445b668", "metadata": {}, "outputs": [], "source": [ "test_data_path = pathlib.Path().resolve().parent.parent / \"testing_python\"" ] }, { "cell_type": "code", "execution_count": 3, "id": "c3753e55", "metadata": {}, "outputs": [], "source": [ "dataset = CODADataset(test_data_path)" ] }, { "cell_type": "code", "execution_count": 4, "id": "b0759a62", "metadata": {}, "outputs": [], "source": [ "training_labels_zarr_store_path = (\n", " dataset.classification_labels_dir_path / \"training_labels.zarr\"\n", ")" ] }, { "cell_type": "code", "execution_count": 5, "id": "8490d25c", "metadata": {}, "outputs": [], "source": [ "root = zarr.open_group(str(training_labels_zarr_store_path), mode=\"w\")\n", "compressor = Blosc(cname='zstd', clevel=5, shuffle=Blosc.BITSHUFFLE)\n", "\n", "def write_group(raw_file: CODARawFile) -> None:\n", " \"\"\"Write a group for a raw file in the training labels Zarr store.\n", " \n", " Args:\n", " raw_file: The CODARawFile instance.\n", " \"\"\"\n", " group = root.create_group(raw_file.filepath.stem, overwrite=True)\n", " chunk_shape = (raw_file.default_chunk_size[1], raw_file.default_chunk_size[0])\n", " for level_number in raw_file.pyramid_level_numbers:\n", " width, height = raw_file.get_image_dimensions_at_level(level_number)\n", " array_shape = (height, width)\n", " arr = group.create_dataset(\n", " name=str(level_number),\n", " shape=array_shape,\n", " chunks=chunk_shape,\n", " dtype=np.uint8,\n", " overwrite=True,\n", " compressor=compressor,\n", " )\n", " arr[:] = 0\n", "\n", "with concurrent.futures.ThreadPoolExecutor() as executor:\n", " executor.map(write_group, dataset.raw_files)" ] }, { "cell_type": "code", "execution_count": null, "id": "4a9272f3", "metadata": {}, "outputs": [], "source": [ "downsample_level = dataset.get_best_downsample_level_for_target_dim(1500)\n", "pad_to = dataset.get_largest_image_dimensions_for_level(downsample_level)\n", "\n", "img_layer_list = []\n", "training_label_layer_list = []\n", "\n", "root = zarr.open_group(str(training_labels_zarr_store_path), mode=\"r\")\n", "\n", "for raw_file in dataset.raw_files:\n", " image_level = raw_file.get_dask_image_level(downsample_level, pad_to=pad_to)\n", " img_layer_list.append(image_level)\n", " raw_label_layer = da.from_zarr(\n", " root[raw_file.filepath.stem][str(downsample_level)]\n", " )\n", " width_pad = pad_to[0] - raw_label_layer.shape[1]\n", " left_pad = width_pad // 2\n", " right_pad = (width_pad // 2) + width_pad % 2\n", " height_pad = pad_to[1] - raw_label_layer.shape[0]\n", " top_pad = height_pad // 2\n", " bot_pad = (height_pad // 2) + height_pad % 2\n", " training_label_layer_list.append(\n", " da.pad(\n", " raw_label_layer,\n", " ((top_pad, bot_pad), (left_pad, right_pad)),\n", " mode=\"constant\",\n", " constant_values=0,\n", " ).rechunk(image_level.chunks[:2])\n", " )" ] }, { "cell_type": "code", "execution_count": null, "id": "e3133630", "metadata": {}, "outputs": [], "source": [ "viewer = napari.Viewer()\n", "image_layer = viewer.add_image(\n", " da.stack(img_layer_list, axis=0),\n", " name=\"dataset\",\n", " contrast_limits=[0, 255],\n", " rgb=True,\n", ")\n", "training_label_layer = viewer.add_labels(\n", " da.stack(training_label_layer_list, axis=0),\n", " name=\"training labels\",\n", ")\n", "napari.run()" ] }, { "cell_type": "code", "execution_count": null, "id": "eb0592ac", "metadata": {}, "outputs": [], "source": [ "example_image_index = random.choice(range(dataset.n_raw_files))\n", "\n", "im_pyramid = dataset.raw_files[example_image_index].get_dask_image_pyramid()\n", "\n", "root = zarr.open_group(str(training_labels_zarr_store_path), mode=\"r\")\n", "label_pyramid = [\n", " da.from_zarr(root[dataset.raw_files[example_image_index].filepath.stem][level])\n", " for level in dataset.raw_files[example_image_index].pyramid_level_numbers\n", "]" ] }, { "cell_type": "code", "execution_count": null, "id": "e9882652", "metadata": {}, "outputs": [], "source": [ "viewer = napari.Viewer()\n", "image_layer = viewer.add_image(\n", " im_pyramid,\n", " name=dataset.raw_files[example_image_index].filepath.stem,\n", " multiscale=True,\n", " contrast_limits=[0, 255],\n", " rgb=True,\n", ")\n", "training_label_layer = viewer.add_labels(\n", " label_pyramid,\n", " name=\"training labels\",\n", " multiscale=True,\n", ")\n", "napari.run()" ] }, { "cell_type": "markdown", "id": "5df414a4", "metadata": {}, "source": [ "#### This works: loading the labels, fully computed, at a scale that doesn't exceed the max texture size" ] }, { "cell_type": "code", "execution_count": 6, "id": "b5a8895b", "metadata": {}, "outputs": [], "source": [ "example_image_index = random.choice(range(dataset.n_raw_files))\n", "\n", "im_pyramid = dataset.raw_files[example_image_index].get_dask_image_pyramid()" ] }, { "cell_type": "code", "execution_count": null, "id": "f0a5a1da", "metadata": {}, "outputs": [], "source": [ "viewer = napari.Viewer()\n", "image_layer = viewer.add_image(\n", " im_pyramid,\n", " name=dataset.raw_files[example_image_index].filepath.stem,\n", " multiscale=True,\n", " contrast_limits=[0, 255],\n", " rgb=True,\n", ")\n", "max_texture_size = glGetIntegerv(GL_MAX_TEXTURE_SIZE)\n", "labels_level = 0\n", "root = zarr.open_group(str(training_labels_zarr_store_path), mode=\"r\")\n", "for level_number in root[\n", " dataset.raw_files[example_image_index].filepath.stem\n", "].array_keys():\n", " label_shape = root[dataset.raw_files[example_image_index].filepath.stem][\n", " level_number\n", " ].shape\n", " if label_shape[1] <= max_texture_size and label_shape[0] <= max_texture_size:\n", " labels_level = int(level_number)\n", " break\n", "labels = da.from_zarr(\n", " root[dataset.raw_files[example_image_index].filepath.stem][str(labels_level)]\n", ").compute()\n", "training_label_layer = viewer.add_labels(\n", " labels,\n", " name=\"training labels\",\n", " scale=(1.0 * 2**labels_level, 1.0 * 2**labels_level),\n", ")\n", "napari.run()" ] }, { "cell_type": "markdown", "id": "96f4f01c", "metadata": {}, "source": [ "#### Writing out the raw tiffs results as multiscale zarrs for testing" ] }, { "cell_type": "code", "execution_count": 9, "id": "72aa66d0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "writing out arrays for lungs_001.ndpi\n", "writing out arrays for lungs_003.ndpi\n", "writing out arrays for lungs_005.ndpi\n", "writing out arrays for lungs_007.ndpi\n", "writing out arrays for lungs_009.ndpi\n", "writing out arrays for lungs_011.ndpi\n", "writing out arrays for lungs_013.ndpi\n", "writing out arrays for lungs_015.ndpi\n", "writing out arrays for lungs_017.ndpi\n", "writing out arrays for lungs_019.ndpi\n", "writing out arrays for lungs_021.ndpi\n", "writing out arrays for lungs_023.ndpi\n", "writing out arrays for lungs_025.ndpi\n", "writing out arrays for lungs_027.ndpi\n", "writing out arrays for lungs_029.ndpi\n", "writing out arrays for lungs_031.ndpi\n", "writing out arrays for lungs_033.ndpi\n", "writing out arrays for lungs_035.ndpi\n", "writing out arrays for lungs_037.ndpi\n", "writing out arrays for lungs_039.ndpi\n" ] } ], "source": [ "\n", "\n", "root = zarr.open_group(str(training_labels_zarr_store_path), mode=\"w\")\n", "compressor = Blosc(cname='zstd', clevel=5, shuffle=Blosc.BITSHUFFLE)\n", "\n", "def write_group_from_tiff(raw_file: CODARawFile) -> None:\n", " \"\"\"Write a group for a raw file in the training labels Zarr store.\n", " \n", " Args:\n", " raw_file: The CODARawFile instance.\n", " \"\"\"\n", " tif_file_path = test_data_path / \"classification_labels\" / f\"{raw_file.filepath.stem}.tif\"\n", " with tifffile.TiffFile(tif_file_path) as tif:\n", " raw_arr = tif.pages[0].asarray()\n", " group = root.create_group(raw_file.filepath.stem, overwrite=True)\n", " chunk_shape = (raw_file.default_chunk_size[1], raw_file.default_chunk_size[0])\n", " for level_number in raw_file.pyramid_level_numbers:\n", " width, height = raw_file.get_image_dimensions_at_level(level_number)\n", " array_shape = (height, width)\n", " arr = group.create_dataset(\n", " name=str(level_number),\n", " shape=array_shape,\n", " chunks=chunk_shape,\n", " dtype=np.uint8,\n", " overwrite=True,\n", " compressor=compressor,\n", " )\n", " arr[:] = cv2.resize(raw_arr, (width, height), interpolation=cv2.INTER_NEAREST)\n", "\n", "# with concurrent.futures.ThreadPoolExecutor() as executor:\n", "# executor.map(write_group_from_tiff, dataset.raw_files)\n", "\n", "for raw_file in dataset.raw_files:\n", " print(f\"writing out arrays for {raw_file.filename}\")\n", " write_group_from_tiff(raw_file)" ] }, { "cell_type": "code", "execution_count": 11, "id": "425b224d", "metadata": {}, "outputs": [], "source": [ "class_names_colors = {\n", " \"bronchioles\": [150, 99, 23, 255], # brown\n", " \"alveoli\": [23, 80, 150, 255], # dark blue\n", " # \"alveoli\": [23, 80, 150, 0], # dark blue\n", " \"vasculature\": [150, 31, 23, 255], # dark red\n", " \"cancer\": [199, 196, 147, 255], # v dark purple\n", " \"nonexpanded\": [23, 80, 150, 255], # dark blue\n", " # \"nonexpanded\": [23, 80, 150, 0], # dark blue\n", " # \"whitespace\": [255, 255, 255, 255], # white\n", " \"whitespace\": [255, 255, 255, 0], # white\n", " \"collagen\": [242, 167, 227, 255], # light pink\n", " # \"collagen\": [242, 167, 227, 0], # light pink\n", "}\n", "\n", "class_names = {i: name for i, name in enumerate(class_names_colors, start=1)}\n", "class_colors = {\n", " i: tuple([v / 255.0 for v in color])\n", " for i, color in enumerate(class_names_colors.values(), start=1)\n", "}" ] }, { "cell_type": "code", "execution_count": 12, "id": "116dfe53", "metadata": {}, "outputs": [], "source": [ "root = zarr.open_group(training_labels_zarr_store_path, mode=\"a\")\n", "\n", "root.attrs[\"class_names\"] = class_names\n", "root.attrs[\"class_colors\"] = class_colors" ] }, { "cell_type": "code", "execution_count": null, "id": "aa5d35b4", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "coda-visual-env", "language": "python", "name": "coda-visual-env" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.9" } }, "nbformat": 4, "nbformat_minor": 5 }