File Layout

This vignette describes the standard file organization for CS9 surveillance systems and provides code templates for each required file in initialization order.

See vignette("installation") for database and environment setup before using these templates. For architecture concepts see vignette("cs9"); for a tutorial on writing tasks see vignette("creating-a-task").

Starting from the example package

The quickest starting point is the complete example:

git clone https://github.com/niphr/cs9example.git
cd cs9example
# Global find/replace "cs9example" with your package name

The sections below cover each required file in initialization order.

00_env_and_namespace.R

https://github.com/niphr/cs9example/blob/main/R/00_env_and_namespace.R

# Environment and namespace configuration
# This file sets up the package environment and namespace exports

# Set environment variable for package data location
Sys.setenv(CS9EXAMPLE_PACKAGE_DIR = system.file("", package = "cs9example"))

# Export functions to namespace
#' @export
get_package_dir <- function() {
  return(Sys.getenv("CS9EXAMPLE_PACKAGE_DIR"))
}

#' @export  
set_config <- function() {
  # Initialize configuration settings
  config <- list()
  return(config)
}

01_definitions.R

https://github.com/niphr/cs9example/blob/main/R/01_definitions.R

# Project-specific definitions
# This file contains global definitions used throughout the surveillance system

#' Set project definitions
#' @export
set_definitions <- function() {
  # Define global variables
  global <- new.env(parent = emptyenv())
  
  # Configuration settings
  global$config <- list()
  global$config$border <- 2020
  global$config$granularity_time <- c("day", "week")
  global$config$granularity_geo <- c("nation", "county", "municip")
  
  # Set in global environment
  assign("global", global, envir = .GlobalEnv)
  
  return(invisible())
}

02_surveillance_systems.R

https://github.com/niphr/cs9example/blob/main/R/02_surveillance_systems.R

# Initialize surveillance systems
# This file creates the main surveillance system objects

#' Set surveillance systems
#' @export
set_surveillance_systems <- function() {
  # Create surveillance system
  global$ss <- cs9::SurveillanceSystem_v9$new(
    name = "cs9example",
    implementation_version = "1.0.0"
  )
  
  return(invisible())
}

03_tables.R

https://github.com/niphr/cs9example/blob/main/R/03_tables.R

# Database table definitions
# This file defines all database tables used by the surveillance system

#' Set database tables
#' @export
set_db_tables <- function() {
  # Weather data table
  global$ss$add_table(
    name_access = "anon",
    name_grouping = "weather",
    name_variant = "data_raw",
    field_types = c(
      "granularity_time" = "TEXT",
      "granularity_geo" = "TEXT",
      "country_iso3" = "TEXT",
      "location_code" = "TEXT",
      "border" = "INTEGER",
      "age" = "TEXT",
      "sex" = "TEXT",
      "date" = "DATE",
      "temp_max" = "DOUBLE",
      "temp_min" = "DOUBLE"
    ),
    keys = c("granularity_time", "location_code", "date", "age", "sex")
  )
  
  return(invisible())
}

04_tasks.R

https://github.com/niphr/cs9example/blob/main/R/04_tasks.R

# Task definitions
# This file defines all surveillance tasks

#' Set surveillance tasks
#' @export
set_tasks <- function() {
  # Weather data download task
  global$ss$add_task(
    name_grouping = "weather",
    name_action = "download_and_import",
    name_variant = "rawdata",
    cores = 1,
    for_each_plan = list(
      location_code = c("norge", "sweden"),
      today = lubridate::today()
    ),
    for_each_analysis = NULL,
    universal_argset = list(
      folder = cs9::path("input", "weather")
    ),
    upsert_at_end_of_each_plan = FALSE,
    insert_at_end_of_each_plan = TRUE,
    action_fn_name = "weather_download_and_import_rawdata_action",
    data_selector_fn_name = "weather_download_and_import_rawdata_data_selector",
    tables = list(
      "anon_weather_data_raw" = "anon_weather_data_raw"
    )
  )
  
  return(invisible())
}

05_deliverables.R

https://github.com/niphr/cs9example/blob/main/R/05_deliverables.R

# Deliverable definitions (optional)
# This file defines outputs and reports generated by the surveillance system

#' Set deliverables
#' @export
set_deliverables <- function() {
  # Example deliverable configuration
  # Can be used to define automated reports, plots, or exports
  
  message("Deliverables configuration loaded")
  return(invisible())
}

10_onLoad.R

https://github.com/niphr/cs9example/blob/main/R/10_onLoad.R

# Package loading sequence
# This file defines what happens when the package is loaded

.onLoad <- function(libname, pkgname) {
  # Set up progress bar handling
  progressr::handlers(progressr::handler_progress(
    format = "[:bar] :current/:total (:percent) in :elapsedfull, eta: :eta",
    clear = FALSE
  ))
  
  # Initialize package components in correct order
  set_definitions()
  set_surveillance_systems() 
  set_db_tables()
  set_tasks()
  
  # Optional: set_deliverables()
  
  invisible()
}

11_onAttach.R

https://github.com/niphr/cs9example/blob/main/R/11_onAttach.R

# Package attachment messages
# This file defines startup messages when package is attached

.onAttach <- function(libname, pkgname) {
  packageStartupMessage("cs9example surveillance system loaded")
  packageStartupMessage("Use global$ss to access the surveillance system")
  packageStartupMessage("Available tasks: ", paste(names(global$ss$tasks), collapse = ", "))
  
  invisible()
}

Task files

Each task has its own .R file named after the task.

weather_download_and_import_rawdata.R

https://github.com/niphr/cs9example/blob/main/R/weather_download_and_import_rawdata.R

# Weather data download and import task
# This file implements the weather data collection task

#' Weather data selector
#' @param argset Named list of arguments
#' @param tables Named list of database tables
#' @return List containing data for processing
#' @export
weather_download_and_import_rawdata_data_selector <- function(argset, tables) {
  # In real implementation, this would download from external API
  # For demonstration, return empty list
  return(list())
}

#' Weather data action
#' @param data Data from data selector function
#' @param argset Named list of arguments
#' @param tables Named list of database tables  
#' @return NULL (side effect: inserts data into database)
#' @export
weather_download_and_import_rawdata_action <- function(data, argset, tables) {
  # Download weather data for specified location and date
  weather_data <- data.table::data.table(
    granularity_time = "day",
    granularity_geo = "nation", 
    country_iso3 = "NOR",
    location_code = argset$location_code,
    border = global$config$border,
    age = "total",
    sex = "total",
    date = argset$today,
    temp_max = runif(1, 15, 25),
    temp_min = runif(1, 5, 15)
  )
  
  # Insert into database
  tables$anon_weather_data_raw$insert_data(weather_data)
  
  return(invisible())
}

weather_clean_data.R

https://github.com/niphr/cs9example/blob/main/R/weather_clean_data.R

# Weather data cleaning task
# This file implements data cleaning and validation

#' Weather cleaning data selector
#' @param argset Named list of arguments
#' @param tables Named list of database tables
#' @return List containing raw weather data
#' @export
weather_clean_data_data_selector <- function(argset, tables) {
  # Get raw weather data from database
  raw_data <- tables$anon_weather_data_raw$tbl() %>%
    dplyr::filter(
      date >= argset$date_from,
      date <= argset$date_to
    ) %>%
    dplyr::collect() %>%
    data.table::as.data.table()
  
  return(list(
    raw_data = raw_data
  ))
}

#' Weather cleaning action
#' @param data Data from data selector function
#' @param argset Named list of arguments
#' @param tables Named list of database tables
#' @return NULL (side effect: processes and validates data)
#' @export
weather_clean_data_action <- function(data, argset, tables) {
  # Apply data cleaning rules
  clean_data <- data$raw_data
  
  # Remove outliers
  clean_data <- clean_data[
    temp_max >= -50 & temp_max <= 60 &
    temp_min >= -60 & temp_min <= 50
  ]
  
  # Add data quality indicators
  clean_data[, data_quality := "good"]
  
  # Log cleaning results
  cs9::update_config_log(
    task = "weather_clean_data",
    msg = paste("Cleaned", nrow(clean_data), "weather records")
  )
  
  return(invisible())
}

weather_export_weather_plots.R

https://github.com/niphr/cs9example/blob/main/R/weather_export_plots.R

# Weather plotting and export task  
# This file implements visualization and reporting

#' Weather plotting data selector
#' @param argset Named list of arguments
#' @param tables Named list of database tables
#' @return List containing weather data for plotting
#' @export
weather_export_plots_data_selector <- function(argset, tables) {
  # Get processed weather data
  plot_data <- tables$anon_weather_data_raw$tbl() %>%
    dplyr::filter(
      location_code == argset$location_code,
      date >= argset$date_from,
      date <= argset$date_to
    ) %>%
    dplyr::collect() %>%
    data.table::as.data.table()
  
  return(list(
    weather_data = plot_data
  ))
}

#' Weather plotting action
#' @param data Data from data selector function
#' @param argset Named list of arguments
#' @param tables Named list of database tables
#' @return NULL (side effect: creates and saves plots)
#' @export
weather_export_plots_action <- function(data, argset, tables) {
  # Create temperature plot
  if(nrow(data$weather_data) > 0) {
    # Generate filename
    filename <- paste0(
      "weather_", argset$location_code, "_", 
      format(Sys.Date(), "%Y%m%d"), ".png"
    )
    
    # Create simple plot
    plot_data <- data$weather_data
    
    p <- ggplot2::ggplot(plot_data, ggplot2::aes(x = date)) +
      ggplot2::geom_line(ggplot2::aes(y = temp_max, color = "Max")) +
      ggplot2::geom_line(ggplot2::aes(y = temp_min, color = "Min")) +
      ggplot2::labs(
        title = paste("Temperature for", argset$location_code),
        x = "Date", y = "Temperature (°C)"
      ) +
      ggplot2::theme_minimal()
    
    # Save plot
    ggplot2::ggsave(
      filename = file.path(argset$output_dir, filename),
      plot = p, width = 10, height = 6, dpi = 150
    )
    
    # Log completion
    cs9::update_config_log(
      task = "weather_export_plots",
      msg = paste("Created plot:", filename)
    )
  }
  
  return(invisible())
}

Notes

  • Initialization order: Files are numbered to match the loading sequence in .onLoad()
  • Global object: global provides shared access to the surveillance system and configuration across files
  • Task files: Each task gets its own file containing the data selector and action functions
  • Table definitions: All database table definitions are centralized in 03_tables.R

Next steps

  • vignette("installation") — set up the database and environment variables
  • vignette("cs9") — architecture and terminology
  • vignette("creating-a-task") — build your first surveillance task