Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions .github/workflows/build-report.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
name: Build report

on:
push:
branches: ["**"]
pull_request:
workflow_dispatch:

jobs:
build-report:
runs-on: ubuntu-latest

steps:
- name: Check out repository
uses: actions/checkout@v4

- name: Set up Pandoc
uses: pandoc/actions/setup@v1

- name: Install XeLaTeX dependencies
run: |
sudo apt-get update
sudo apt-get install -y \
texlive-xetex \
texlive-latex-extra \
texlive-fonts-recommended \
texlive-bibtex-extra \
latexmk

- name: Verify tool versions
run: |
pandoc --version
xelatex --version

- name: Inspect TeX packages
run: |
kpsewhich hyperref.sty || true
kpsewhich hyperxmp.sty || true
kpsewhich bookmark.sty || true
xelatex --version || true

- name: Build report artifacts
id: build_report
continue-on-error: true
run: make all

- name: Upload PDF artifact
uses: actions/upload-artifact@v4
with:
name: spatial-ninjas-report-pdf
path: build/spatial-ninjas-report.pdf
if-no-files-found: warn

- name: Upload HTML artifact
uses: actions/upload-artifact@v4
with:
name: spatial-ninjas-report-html
path: build/spatial-ninjas-report.html
if-no-files-found: warn

- name: Upload merged markdown artifact
uses: actions/upload-artifact@v4
with:
name: spatial-ninjas-report-merged-markdown
path: build/merged.md
if-no-files-found: warn

- name: Fail job if build failed
if: ${{ steps.build_report.outcome == 'failure' }}
run: |
echo "make all failed"
exit 1
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
.DS_Store
build/
94 changes: 94 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
# ---------- Config ----------
PANDOC := pandoc
PDF_ENGINE := xelatex

BUILD_DIR := build
DOCS_DIR := docs
TEMPLATE_DIR := template

METADATA := report-metadata.yaml
HEADER := $(TEMPLATE_DIR)/header.tex
BIB_FILE := references.bib
CSL_FILE := $(TEMPLATE_DIR)/ieee.csl

MERGED_MD := $(BUILD_DIR)/merged.md
PDF_OUTPUT := $(BUILD_DIR)/spatial-ninjas-report.pdf
HTML_OUTPUT := $(BUILD_DIR)/spatial-ninjas-report.html

# ---------- Source discovery ----------
FRONTMATTER := $(sort $(wildcard $(DOCS_DIR)/00-frontmatter/*.md))
MEMBER_NOTES := $(sort $(wildcard $(DOCS_DIR)/members/*/*.md))
SHARED_NOTES := $(sort $(wildcard $(DOCS_DIR)/shared/*.md))
BACKMATTER := $(sort $(wildcard $(DOCS_DIR)/99-backmatter/*.md))

SOURCES := $(FRONTMATTER) $(MEMBER_NOTES) $(SHARED_NOTES) $(BACKMATTER)

# ---------- Pandoc flags ----------
PANDOC_COMMON_FLAGS := \
--from=markdown \
--standalone \
--citeproc \
--metadata-file=$(METADATA) \
--bibliography=$(BIB_FILE) \
--metadata=date:"$(shell date "+%B %-d, %Y")" \
--csl=$(CSL_FILE) \
--resource-path=.:$(DOCS_DIR):$(TEMPLATE_DIR):$(BUILD_DIR) \
--toc \
--number-sections

PANDOC_PDF_FLAGS := \
$(PANDOC_COMMON_FLAGS) \
--include-in-header=$(HEADER)

PANDOC_HTML_FLAGS := \
$(PANDOC_COMMON_FLAGS)

# ---------- Targets ----------
.PHONY: help all pdf html merge clean sources check

help: ## Show available make targets
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
awk 'BEGIN {FS = ":.*?## "}; {printf "%-15s %s\n", $$1, $$2}'

all: pdf html ## Build all output formats

check: ## Verify required files exist
@test -f $(METADATA) || (echo "Missing $(METADATA)"; exit 1)
@test -f $(HEADER) || (echo "Missing $(HEADER)"; exit 1)
@test -f $(BIB_FILE) || (echo "Missing $(BIB_FILE)"; exit 1)
@test -f $(CSL_FILE) || (echo "Missing $(CSL_FILE)"; exit 1)

pdf: $(PDF_OUTPUT) ## Build PDF report

$(PDF_OUTPUT): check $(MERGED_MD) $(METADATA) $(HEADER) $(BIB_FILE) $(CSL_FILE)
$(PANDOC) $(MERGED_MD) \
$(PANDOC_PDF_FLAGS) \
--pdf-engine=$(PDF_ENGINE) \
-o $(PDF_OUTPUT)

html: $(HTML_OUTPUT) ## Build HTML report

$(HTML_OUTPUT): check $(MERGED_MD) $(METADATA) $(BIB_FILE) $(CSL_FILE)
$(PANDOC) $(MERGED_MD) \
$(PANDOC_HTML_FLAGS) \
-o $(HTML_OUTPUT)

merge: $(MERGED_MD) ## Merge markdown sources into a single file

$(MERGED_MD): $(SOURCES) | $(BUILD_DIR)
@echo "Merging markdown sources..."
@rm -f $(MERGED_MD)
@for f in $(SOURCES); do \
echo "Adding $$f"; \
cat $$f >> $(MERGED_MD); \
printf '\n\n' >> $(MERGED_MD); \
done

$(BUILD_DIR):
mkdir -p $(BUILD_DIR)

sources: ## Print detected markdown source files
@printf '%s\n' $(SOURCES)

clean: ## Remove build artifacts
rm -rf $(BUILD_DIR)
99 changes: 90 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,95 @@ This repository contains the research workspace for the project

The repository will host materials related to:

- literature reviews and research notes
- experiment code
- benchmark implementations
- datasets used for evaluation
- research notes and documentation
- references and literature summaries
- documentation and project outputs

## Structure

The repository will gradually be organized into the following directories:
## Report Build System

The project report is written in **Markdown** and compiled using **Pandoc**.

A `Makefile` is provided to automate the process of merging all documentation and generating the final outputs.

## Build the report

To build all report formats:

```bash
make all
````

This generates:

```
build/spatial-ninjas-report.pdf
build/spatial-ninjas-report.html
```

You can also build individual formats:

```bash
make pdf
make html
```
docs/ project documentation, research notes, and project log
experiments/ experiment scripts and notebooks
benchmarks/ spatial reasoning benchmark implementations
datasets/ datasets used for experiments
scripts/ utilities and evaluation scripts

Other useful commands:

```bash
make merge # merge markdown sources into build/merged.md
make sources # list detected markdown source files
make clean # remove build artifacts
make help # list available targets
```

The build pipeline:

1. discovers markdown files in the `docs/` directory
2. merges them into a single document
3. runs Pandoc to produce PDF and HTML outputs
4. formats citations using the IEEE CSL style


## Repository Structure

```
docs/ project documentation and literature summaries
00-frontmatter/ report introduction and overview
members/ paper summaries written by individual members
shared/ synthesis and next steps written collaboratively
99-backmatter/ references and appendix
template/ Pandoc LaTeX header and citation style
references.bib shared bibliography database
report-metadata.yaml document metadata used by Pandoc
Makefile report build automation
```

### Member summaries

Each project member writes their paper summary in:

```
docs/members/<name>/
```

These are automatically included in the report during the build process.

### Shared sections

Collaborative sections are stored in:

```
docs/shared/
```

Examples include:

* literature synthesis
* proposed next steps for experiments


## References

Expand All @@ -32,6 +104,15 @@ Bibliographic references used in the project are stored in:
references.bib
```

Citations inside markdown files use standard Pandoc citation syntax:

```
@paper_key
```

The final report is formatted using the **IEEE citation style**.


## Project Management

Task planning and sprint tracking are handled via the project board:
Expand Down
Empty file added docs/00-frontmatter/overview.md
Empty file.
3 changes: 3 additions & 0 deletions docs/00-frontmatter/title.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Overview

This document collects the Spatial Ninjas literature summaries, synthesis notes, and next steps.
4 changes: 4 additions & 0 deletions docs/99-backmatter/appendix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# References {.unnumbered}

::: {#refs}
:::
13 changes: 13 additions & 0 deletions docs/members/eemil/multi-task-benchmark.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# LLMs on Geospatial Tasks: Multi-Task Benchmarking

## Summary

Eemil's work. Write something here @xu_evaluating_2025.

## Main contribution

...

## Relevance to our project

...
13 changes: 13 additions & 0 deletions docs/members/ki-chun/minds-eye.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Mind's Eye of LLMs

## Summary

Ki Chun's work. Write something here @wu_mind_2024.

## Main contribution

...

## Relevance to our project

...
13 changes: 13 additions & 0 deletions docs/members/oliver/spatial-cognition-llms.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Spatial Cognition Abilities of LLMs

## Summary

Oliver's work. Write something here @yang_evaluating_2025.

## Main contribution

...

## Relevance to our project

...
17 changes: 17 additions & 0 deletions docs/members/pawel/geollm.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# GeoLLM: Extracting Geospatial Knowledge from LLMs

## Summary

Manvi et al. uses a programmatic aproach to test the LLM.
In their code they amongst others generate a large JSON file with prompts with coordinates and geographical names.
They then ask the LLM with a prefix prompt to answer the questions with a certain number, and they can then evaluate these compared with actual answers from coordinates on an actual map. One of their evaluation methods is the Spearmanr correlation and finding Pearsonr correlation and r^2 value.

I had written a modified and simplified code based on their source code but a lighter version which takes their existing prompts file and maps, it generates a CSV file with prompt anwers which are then evaluated by the Spearmanr correlation. So far the codes were tested on Gemma 3, while the plan is to test it on Gemini 3.1 Pro, GPT 5.2, DeepSeek and Llama3 as a open source backup.

## Main contribution

...

## Relevance to our project

...
13 changes: 13 additions & 0 deletions docs/members/topi/llm-geotextcog.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# LLM-GeoTextCog: A Cognitive Enhancement Framework

## Summary

Topi's work. Write something here @wang_llm-geotextcog_2026.

## Main contribution

...

## Relevance to our project

...
13 changes: 13 additions & 0 deletions docs/members/totti/chatgpt-geospatial-skills.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Geospatial Skills of ChatGPT

## Summary

Totti's work. Write something here @mooney_towards_2023.

## Main contribution

...

## Relevance to our project

...
Loading
Loading