---
title: "File Layout"
output: rmarkdown::html_vignette
vignette: >
  %\VignetteIndexEntry{File Layout}
  %\VignetteEncoding{UTF-8}
  %\VignetteEngine{knitr::rmarkdown}
editor_options:
  chunk_output_type: console
---



This vignette describes the standard file organization for CS9 surveillance systems and provides code templates for each required file in initialization order.

See `vignette("installation")` for database and environment setup before using these templates. For architecture concepts see `vignette("cs9")`; for a tutorial on writing tasks see `vignette("creating-a-task")`.

## Starting from the example package

The quickest starting point is the complete example:

```bash
git clone https://github.com/niphr/cs9example.git
cd cs9example
# Global find/replace "cs9example" with your package name
```

The sections below cover each required file in initialization order.

## 00_env_and_namespace.R

https://github.com/niphr/cs9example/blob/main/R/00_env_and_namespace.R

```r
# Environment and namespace configuration
# This file sets up the package environment and namespace exports

# Set environment variable for package data location
Sys.setenv(CS9EXAMPLE_PACKAGE_DIR = system.file("", package = "cs9example"))

# Export functions to namespace
#' @export
get_package_dir <- function() {
  return(Sys.getenv("CS9EXAMPLE_PACKAGE_DIR"))
}

#' @export  
set_config <- function() {
  # Initialize configuration settings
  config <- list()
  return(config)
}
```

## 01_definitions.R

https://github.com/niphr/cs9example/blob/main/R/01_definitions.R

```r
# Project-specific definitions
# This file contains global definitions used throughout the surveillance system

#' Set project definitions
#' @export
set_definitions <- function() {
  # Define global variables
  global <- new.env(parent = emptyenv())
  
  # Configuration settings
  global$config <- list()
  global$config$border <- 2020
  global$config$granularity_time <- c("day", "week")
  global$config$granularity_geo <- c("nation", "county", "municip")
  
  # Set in global environment
  assign("global", global, envir = .GlobalEnv)
  
  return(invisible())
}
```

## 02_surveillance_systems.R

https://github.com/niphr/cs9example/blob/main/R/02_surveillance_systems.R

```r
# Initialize surveillance systems
# This file creates the main surveillance system objects

#' Set surveillance systems
#' @export
set_surveillance_systems <- function() {
  # Create surveillance system
  global$ss <- cs9::SurveillanceSystem_v9$new(
    name = "cs9example",
    implementation_version = "1.0.0"
  )
  
  return(invisible())
}
```

## 03_tables.R

https://github.com/niphr/cs9example/blob/main/R/03_tables.R

```r
# Database table definitions
# This file defines all database tables used by the surveillance system

#' Set database tables
#' @export
set_db_tables <- function() {
  # Weather data table
  global$ss$add_table(
    name_access = "anon",
    name_grouping = "weather",
    name_variant = "data_raw",
    field_types = c(
      "granularity_time" = "TEXT",
      "granularity_geo" = "TEXT",
      "country_iso3" = "TEXT",
      "location_code" = "TEXT",
      "border" = "INTEGER",
      "age" = "TEXT",
      "sex" = "TEXT",
      "date" = "DATE",
      "temp_max" = "DOUBLE",
      "temp_min" = "DOUBLE"
    ),
    keys = c("granularity_time", "location_code", "date", "age", "sex")
  )
  
  return(invisible())
}
```

## 04_tasks.R

https://github.com/niphr/cs9example/blob/main/R/04_tasks.R

```r
# Task definitions
# This file defines all surveillance tasks

#' Set surveillance tasks
#' @export
set_tasks <- function() {
  # Weather data download task
  global$ss$add_task(
    name_grouping = "weather",
    name_action = "download_and_import",
    name_variant = "rawdata",
    cores = 1,
    for_each_plan = list(
      location_code = c("norge", "sweden"),
      today = lubridate::today()
    ),
    for_each_analysis = NULL,
    universal_argset = list(
      folder = cs9::path("input", "weather")
    ),
    upsert_at_end_of_each_plan = FALSE,
    insert_at_end_of_each_plan = TRUE,
    action_fn_name = "weather_download_and_import_rawdata_action",
    data_selector_fn_name = "weather_download_and_import_rawdata_data_selector",
    tables = list(
      "anon_weather_data_raw" = "anon_weather_data_raw"
    )
  )
  
  return(invisible())
}
```

## 05_deliverables.R

https://github.com/niphr/cs9example/blob/main/R/05_deliverables.R

```r
# Deliverable definitions (optional)
# This file defines outputs and reports generated by the surveillance system

#' Set deliverables
#' @export
set_deliverables <- function() {
  # Example deliverable configuration
  # Can be used to define automated reports, plots, or exports
  
  message("Deliverables configuration loaded")
  return(invisible())
}
```

## 10_onLoad.R

https://github.com/niphr/cs9example/blob/main/R/10_onLoad.R

```r
# Package loading sequence
# This file defines what happens when the package is loaded

.onLoad <- function(libname, pkgname) {
  # Set up progress bar handling
  progressr::handlers(progressr::handler_progress(
    format = "[:bar] :current/:total (:percent) in :elapsedfull, eta: :eta",
    clear = FALSE
  ))
  
  # Initialize package components in correct order
  set_definitions()
  set_surveillance_systems() 
  set_db_tables()
  set_tasks()
  
  # Optional: set_deliverables()
  
  invisible()
}
```

## 11_onAttach.R

https://github.com/niphr/cs9example/blob/main/R/11_onAttach.R

```r
# Package attachment messages
# This file defines startup messages when package is attached

.onAttach <- function(libname, pkgname) {
  packageStartupMessage("cs9example surveillance system loaded")
  packageStartupMessage("Use global$ss to access the surveillance system")
  packageStartupMessage("Available tasks: ", paste(names(global$ss$tasks), collapse = ", "))
  
  invisible()
}
```

## Task files

Each task has its own `.R` file named after the task.

### weather_download_and_import_rawdata.R

https://github.com/niphr/cs9example/blob/main/R/weather_download_and_import_rawdata.R

```r
# Weather data download and import task
# This file implements the weather data collection task

#' Weather data selector
#' @param argset Named list of arguments
#' @param tables Named list of database tables
#' @return List containing data for processing
#' @export
weather_download_and_import_rawdata_data_selector <- function(argset, tables) {
  # In real implementation, this would download from external API
  # For demonstration, return empty list
  return(list())
}

#' Weather data action
#' @param data Data from data selector function
#' @param argset Named list of arguments
#' @param tables Named list of database tables  
#' @return NULL (side effect: inserts data into database)
#' @export
weather_download_and_import_rawdata_action <- function(data, argset, tables) {
  # Download weather data for specified location and date
  weather_data <- data.table::data.table(
    granularity_time = "day",
    granularity_geo = "nation", 
    country_iso3 = "NOR",
    location_code = argset$location_code,
    border = global$config$border,
    age = "total",
    sex = "total",
    date = argset$today,
    temp_max = runif(1, 15, 25),
    temp_min = runif(1, 5, 15)
  )
  
  # Insert into database
  tables$anon_weather_data_raw$insert_data(weather_data)
  
  return(invisible())
}
```

### weather_clean_data.R

https://github.com/niphr/cs9example/blob/main/R/weather_clean_data.R
```r
# Weather data cleaning task
# This file implements data cleaning and validation

#' Weather cleaning data selector
#' @param argset Named list of arguments
#' @param tables Named list of database tables
#' @return List containing raw weather data
#' @export
weather_clean_data_data_selector <- function(argset, tables) {
  # Get raw weather data from database
  raw_data <- tables$anon_weather_data_raw$tbl() %>%
    dplyr::filter(
      date >= argset$date_from,
      date <= argset$date_to
    ) %>%
    dplyr::collect() %>%
    data.table::as.data.table()
  
  return(list(
    raw_data = raw_data
  ))
}

#' Weather cleaning action
#' @param data Data from data selector function
#' @param argset Named list of arguments
#' @param tables Named list of database tables
#' @return NULL (side effect: processes and validates data)
#' @export
weather_clean_data_action <- function(data, argset, tables) {
  # Apply data cleaning rules
  clean_data <- data$raw_data
  
  # Remove outliers
  clean_data <- clean_data[
    temp_max >= -50 & temp_max <= 60 &
    temp_min >= -60 & temp_min <= 50
  ]
  
  # Add data quality indicators
  clean_data[, data_quality := "good"]
  
  # Log cleaning results
  cs9::update_config_log(
    task = "weather_clean_data",
    msg = paste("Cleaned", nrow(clean_data), "weather records")
  )
  
  return(invisible())
}
```

### weather_export_weather_plots.R

https://github.com/niphr/cs9example/blob/main/R/weather_export_plots.R

```r
# Weather plotting and export task  
# This file implements visualization and reporting

#' Weather plotting data selector
#' @param argset Named list of arguments
#' @param tables Named list of database tables
#' @return List containing weather data for plotting
#' @export
weather_export_plots_data_selector <- function(argset, tables) {
  # Get processed weather data
  plot_data <- tables$anon_weather_data_raw$tbl() %>%
    dplyr::filter(
      location_code == argset$location_code,
      date >= argset$date_from,
      date <= argset$date_to
    ) %>%
    dplyr::collect() %>%
    data.table::as.data.table()
  
  return(list(
    weather_data = plot_data
  ))
}

#' Weather plotting action
#' @param data Data from data selector function
#' @param argset Named list of arguments
#' @param tables Named list of database tables
#' @return NULL (side effect: creates and saves plots)
#' @export
weather_export_plots_action <- function(data, argset, tables) {
  # Create temperature plot
  if(nrow(data$weather_data) > 0) {
    # Generate filename
    filename <- paste0(
      "weather_", argset$location_code, "_", 
      format(Sys.Date(), "%Y%m%d"), ".png"
    )
    
    # Create simple plot
    plot_data <- data$weather_data
    
    p <- ggplot2::ggplot(plot_data, ggplot2::aes(x = date)) +
      ggplot2::geom_line(ggplot2::aes(y = temp_max, color = "Max")) +
      ggplot2::geom_line(ggplot2::aes(y = temp_min, color = "Min")) +
      ggplot2::labs(
        title = paste("Temperature for", argset$location_code),
        x = "Date", y = "Temperature (°C)"
      ) +
      ggplot2::theme_minimal()
    
    # Save plot
    ggplot2::ggsave(
      filename = file.path(argset$output_dir, filename),
      plot = p, width = 10, height = 6, dpi = 150
    )
    
    # Log completion
    cs9::update_config_log(
      task = "weather_export_plots",
      msg = paste("Created plot:", filename)
    )
  }
  
  return(invisible())
}
```

## Notes

- **Initialization order**: Files are numbered to match the loading sequence in `.onLoad()`
- **Global object**: `global` provides shared access to the surveillance system and configuration across files
- **Task files**: Each task gets its own file containing the data selector and action functions
- **Table definitions**: All database table definitions are centralized in `03_tables.R`

## Next steps

- `vignette("installation")` — set up the database and environment variables
- `vignette("cs9")` — architecture and terminology
- `vignette("creating-a-task")` — build your first surveillance task