diff --git a/.github/workflows/build-report.yml b/.github/workflows/build-report.yml new file mode 100644 index 0000000..099708e --- /dev/null +++ b/.github/workflows/build-report.yml @@ -0,0 +1,72 @@ +name: Build report + +on: + push: + branches: ["**"] + pull_request: + workflow_dispatch: + +jobs: + build-report: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Pandoc + uses: pandoc/actions/setup@v1 + + - name: Install XeLaTeX dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + texlive-xetex \ + texlive-latex-extra \ + texlive-fonts-recommended \ + texlive-bibtex-extra \ + latexmk + + - name: Verify tool versions + run: | + pandoc --version + xelatex --version + + - name: Inspect TeX packages + run: | + kpsewhich hyperref.sty || true + kpsewhich hyperxmp.sty || true + kpsewhich bookmark.sty || true + xelatex --version || true + + - name: Build report artifacts + id: build_report + continue-on-error: true + run: make all + + - name: Upload PDF artifact + uses: actions/upload-artifact@v4 + with: + name: spatial-ninjas-report-pdf + path: build/spatial-ninjas-report.pdf + if-no-files-found: warn + + - name: Upload HTML artifact + uses: actions/upload-artifact@v4 + with: + name: spatial-ninjas-report-html + path: build/spatial-ninjas-report.html + if-no-files-found: warn + + - name: Upload merged markdown artifact + uses: actions/upload-artifact@v4 + with: + name: spatial-ninjas-report-merged-markdown + path: build/merged.md + if-no-files-found: warn + + - name: Fail job if build failed + if: ${{ steps.build_report.outcome == 'failure' }} + run: | + echo "make all failed" + exit 1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..aaab5ad --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.DS_Store +build/ \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8005370 --- /dev/null +++ b/Makefile @@ -0,0 +1,94 @@ +# ---------- Config ---------- +PANDOC := pandoc +PDF_ENGINE := xelatex + +BUILD_DIR := build +DOCS_DIR := docs +TEMPLATE_DIR := template + +METADATA := report-metadata.yaml +HEADER := $(TEMPLATE_DIR)/header.tex +BIB_FILE := references.bib +CSL_FILE := $(TEMPLATE_DIR)/ieee.csl + +MERGED_MD := $(BUILD_DIR)/merged.md +PDF_OUTPUT := $(BUILD_DIR)/spatial-ninjas-report.pdf +HTML_OUTPUT := $(BUILD_DIR)/spatial-ninjas-report.html + +# ---------- Source discovery ---------- +FRONTMATTER := $(sort $(wildcard $(DOCS_DIR)/00-frontmatter/*.md)) +MEMBER_NOTES := $(sort $(wildcard $(DOCS_DIR)/members/*/*.md)) +SHARED_NOTES := $(sort $(wildcard $(DOCS_DIR)/shared/*.md)) +BACKMATTER := $(sort $(wildcard $(DOCS_DIR)/99-backmatter/*.md)) + +SOURCES := $(FRONTMATTER) $(MEMBER_NOTES) $(SHARED_NOTES) $(BACKMATTER) + +# ---------- Pandoc flags ---------- +PANDOC_COMMON_FLAGS := \ + --from=markdown \ + --standalone \ + --citeproc \ + --metadata-file=$(METADATA) \ + --bibliography=$(BIB_FILE) \ + --metadata=date:"$(shell date "+%B %-d, %Y")" \ + --csl=$(CSL_FILE) \ + --resource-path=.:$(DOCS_DIR):$(TEMPLATE_DIR):$(BUILD_DIR) \ + --toc \ + --number-sections + +PANDOC_PDF_FLAGS := \ + $(PANDOC_COMMON_FLAGS) \ + --include-in-header=$(HEADER) + +PANDOC_HTML_FLAGS := \ + $(PANDOC_COMMON_FLAGS) + +# ---------- Targets ---------- +.PHONY: help all pdf html merge clean sources check + +help: ## Show available make targets + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf "%-15s %s\n", $$1, $$2}' + +all: pdf html ## Build all output formats + +check: ## Verify required files exist + @test -f $(METADATA) || (echo "Missing $(METADATA)"; exit 1) + @test -f $(HEADER) || (echo "Missing $(HEADER)"; exit 1) + @test -f $(BIB_FILE) || (echo "Missing $(BIB_FILE)"; exit 1) + @test -f $(CSL_FILE) || (echo "Missing $(CSL_FILE)"; exit 1) + +pdf: $(PDF_OUTPUT) ## Build PDF report + +$(PDF_OUTPUT): check $(MERGED_MD) $(METADATA) $(HEADER) $(BIB_FILE) $(CSL_FILE) + $(PANDOC) $(MERGED_MD) \ + $(PANDOC_PDF_FLAGS) \ + --pdf-engine=$(PDF_ENGINE) \ + -o $(PDF_OUTPUT) + +html: $(HTML_OUTPUT) ## Build HTML report + +$(HTML_OUTPUT): check $(MERGED_MD) $(METADATA) $(BIB_FILE) $(CSL_FILE) + $(PANDOC) $(MERGED_MD) \ + $(PANDOC_HTML_FLAGS) \ + -o $(HTML_OUTPUT) + +merge: $(MERGED_MD) ## Merge markdown sources into a single file + +$(MERGED_MD): $(SOURCES) | $(BUILD_DIR) + @echo "Merging markdown sources..." + @rm -f $(MERGED_MD) + @for f in $(SOURCES); do \ + echo "Adding $$f"; \ + cat $$f >> $(MERGED_MD); \ + printf '\n\n' >> $(MERGED_MD); \ + done + +$(BUILD_DIR): + mkdir -p $(BUILD_DIR) + +sources: ## Print detected markdown source files + @printf '%s\n' $(SOURCES) + +clean: ## Remove build artifacts + rm -rf $(BUILD_DIR) diff --git a/README.md b/README.md index c6719b2..ee56aa7 100644 --- a/README.md +++ b/README.md @@ -6,23 +6,95 @@ This repository contains the research workspace for the project The repository will host materials related to: +- literature reviews and research notes - experiment code - benchmark implementations - datasets used for evaluation -- research notes and documentation -- references and literature summaries +- documentation and project outputs -## Structure -The repository will gradually be organized into the following directories: +## Report Build System +The project report is written in **Markdown** and compiled using **Pandoc**. + +A `Makefile` is provided to automate the process of merging all documentation and generating the final outputs. + +## Build the report + +To build all report formats: + +```bash +make all +```` + +This generates: + +``` +build/spatial-ninjas-report.pdf +build/spatial-ninjas-report.html +``` + +You can also build individual formats: + +```bash +make pdf +make html ``` -docs/ project documentation, research notes, and project log -experiments/ experiment scripts and notebooks -benchmarks/ spatial reasoning benchmark implementations -datasets/ datasets used for experiments -scripts/ utilities and evaluation scripts + +Other useful commands: + +```bash +make merge # merge markdown sources into build/merged.md +make sources # list detected markdown source files +make clean # remove build artifacts +make help # list available targets +``` + +The build pipeline: + +1. discovers markdown files in the `docs/` directory +2. merges them into a single document +3. runs Pandoc to produce PDF and HTML outputs +4. formats citations using the IEEE CSL style + + +## Repository Structure + ``` +docs/ project documentation and literature summaries + 00-frontmatter/ report introduction and overview + members/ paper summaries written by individual members + shared/ synthesis and next steps written collaboratively + 99-backmatter/ references and appendix +template/ Pandoc LaTeX header and citation style +references.bib shared bibliography database +report-metadata.yaml document metadata used by Pandoc +Makefile report build automation +``` + +### Member summaries + +Each project member writes their paper summary in: + +``` +docs/members// +``` + +These are automatically included in the report during the build process. + +### Shared sections + +Collaborative sections are stored in: + +``` +docs/shared/ +``` + +Examples include: + +* literature synthesis +* proposed next steps for experiments + ## References @@ -32,6 +104,15 @@ Bibliographic references used in the project are stored in: references.bib ``` +Citations inside markdown files use standard Pandoc citation syntax: + +``` +@paper_key +``` + +The final report is formatted using the **IEEE citation style**. + + ## Project Management Task planning and sprint tracking are handled via the project board: diff --git a/docs/00-frontmatter/overview.md b/docs/00-frontmatter/overview.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/00-frontmatter/title.md b/docs/00-frontmatter/title.md new file mode 100644 index 0000000..1798d82 --- /dev/null +++ b/docs/00-frontmatter/title.md @@ -0,0 +1,3 @@ +# Overview + +This document collects the Spatial Ninjas literature summaries, synthesis notes, and next steps. \ No newline at end of file diff --git a/docs/99-backmatter/appendix.md b/docs/99-backmatter/appendix.md new file mode 100644 index 0000000..4cdf317 --- /dev/null +++ b/docs/99-backmatter/appendix.md @@ -0,0 +1,4 @@ +# References {.unnumbered} + +::: {#refs} +::: \ No newline at end of file diff --git a/docs/members/eemil/multi-task-benchmark.md b/docs/members/eemil/multi-task-benchmark.md new file mode 100644 index 0000000..02faa5e --- /dev/null +++ b/docs/members/eemil/multi-task-benchmark.md @@ -0,0 +1,13 @@ +# LLMs on Geospatial Tasks: Multi-Task Benchmarking + +## Summary + +Eemil's work. Write something here @xu_evaluating_2025. + +## Main contribution + +... + +## Relevance to our project + +... diff --git a/docs/members/ki-chun/minds-eye.md b/docs/members/ki-chun/minds-eye.md new file mode 100644 index 0000000..cae1e57 --- /dev/null +++ b/docs/members/ki-chun/minds-eye.md @@ -0,0 +1,13 @@ +# Mind's Eye of LLMs + +## Summary + +Ki Chun's work. Write something here @wu_mind_2024. + +## Main contribution + +... + +## Relevance to our project + +... diff --git a/docs/members/oliver/spatial-cognition-llms.md b/docs/members/oliver/spatial-cognition-llms.md new file mode 100644 index 0000000..b5106af --- /dev/null +++ b/docs/members/oliver/spatial-cognition-llms.md @@ -0,0 +1,13 @@ +# Spatial Cognition Abilities of LLMs + +## Summary + +Oliver's work. Write something here @yang_evaluating_2025. + +## Main contribution + +... + +## Relevance to our project + +... diff --git a/docs/members/pawel/geollm.md b/docs/members/pawel/geollm.md new file mode 100644 index 0000000..98643d8 --- /dev/null +++ b/docs/members/pawel/geollm.md @@ -0,0 +1,17 @@ +# GeoLLM: Extracting Geospatial Knowledge from LLMs + +## Summary + +Manvi et al. uses a programmatic aproach to test the LLM. +In their code they amongst others generate a large JSON file with prompts with coordinates and geographical names. +They then ask the LLM with a prefix prompt to answer the questions with a certain number, and they can then evaluate these compared with actual answers from coordinates on an actual map. One of their evaluation methods is the Spearmanr correlation and finding Pearsonr correlation and r^2 value. + +I had written a modified and simplified code based on their source code but a lighter version which takes their existing prompts file and maps, it generates a CSV file with prompt anwers which are then evaluated by the Spearmanr correlation. So far the codes were tested on Gemma 3, while the plan is to test it on Gemini 3.1 Pro, GPT 5.2, DeepSeek and Llama3 as a open source backup. + +## Main contribution + +... + +## Relevance to our project + +... diff --git a/docs/members/topi/llm-geotextcog.md b/docs/members/topi/llm-geotextcog.md new file mode 100644 index 0000000..c99d7e3 --- /dev/null +++ b/docs/members/topi/llm-geotextcog.md @@ -0,0 +1,13 @@ +# LLM-GeoTextCog: A Cognitive Enhancement Framework + +## Summary + +Topi's work. Write something here @wang_llm-geotextcog_2026. + +## Main contribution + +... + +## Relevance to our project + +... diff --git a/docs/members/totti/chatgpt-geospatial-skills.md b/docs/members/totti/chatgpt-geospatial-skills.md new file mode 100644 index 0000000..1790b81 --- /dev/null +++ b/docs/members/totti/chatgpt-geospatial-skills.md @@ -0,0 +1,13 @@ +# Geospatial Skills of ChatGPT + +## Summary + +Totti's work. Write something here @mooney_towards_2023. + +## Main contribution + +... + +## Relevance to our project + +... diff --git a/docs/shared/next-steps.md b/docs/shared/next-steps.md new file mode 100644 index 0000000..e0c0bc4 --- /dev/null +++ b/docs/shared/next-steps.md @@ -0,0 +1,32 @@ +# Next Steps + +## Candidate evaluation tasks + +Potential tasks for benchmarking spatial reasoning: + +- distance estimation between cities +- direction inference ("Which city lies north of...") +- route description tasks + +## Models to evaluate + +Possible models to include: + +- GPT-4o +- Claude +- Gemini +- Llama + +## Enhancement approaches to test + +Based on the literature: + +- OpenStreetMap augmentation (GeoLLM) +- Visualization-of-Thought prompting +- tool-assisted spatial computation + +## Immediate implementation tasks + +- build evaluation prompt templates +- implement benchmark runner +- design scoring metrics diff --git a/docs/shared/synthesis.md b/docs/shared/synthesis.md new file mode 100644 index 0000000..d341b6c --- /dev/null +++ b/docs/shared/synthesis.md @@ -0,0 +1,26 @@ +# Literature Synthesis + +## Categories of spatial reasoning tasks + +Across the surveyed papers, spatial reasoning tasks can be grouped into: + +- distance estimation +- direction inference +- topological relationships +- map-based question answering + +## Evaluation approaches + +Three main evaluation approaches appear: + +- benchmark datasets (Xu et al.) +- prompting techniques (Wu et al.) +- external knowledge integration (Manvi et al.) + +## Common weaknesses of LLMs + +Several studies report similar failure modes: + +- inconsistent distance estimation +- confusion between relative directions +- hallucinated geographic facts diff --git a/report-metadata.yaml b/report-metadata.yaml new file mode 100644 index 0000000..d50e83b --- /dev/null +++ b/report-metadata.yaml @@ -0,0 +1,29 @@ +title: "LLM Spatial Reasoning: Evaluating and Enhancing Geographic Cognition in Language Models" +subtitle: "Spatial Ninjas — Course Project Report" + +author: + - Ki Chun Tong + - Pawel Dzierzynski + - Oliver Isaksson + - Eemil Koskinen + - Topi Nieminen + - Totti L. + +date: $date$ +lang: en + +keywords: + - spatial reasoning + - large language models + - geospatial cognition + - GeoAI + +abstract: | + This report evaluates the spatial reasoning capabilities of large language + models and explores methods to improve geographic cognition in LLM-based + systems. + +link-citations: true +toc: true +toc-depth: 2 +number-sections: true \ No newline at end of file diff --git a/template/header.tex b/template/header.tex new file mode 100644 index 0000000..43463fd --- /dev/null +++ b/template/header.tex @@ -0,0 +1,21 @@ +\usepackage[margin=1in]{geometry} +\usepackage{parskip} +\setlength{\parindent}{0pt} +\usepackage{graphicx} +\usepackage{booktabs} +\usepackage{longtable} +\usepackage{xcolor} +\usepackage{hyperref} +\usepackage{microtype} + +\hypersetup{ + colorlinks=true, + linkcolor=blue, + urlcolor=blue, + citecolor=blue +} + +\usepackage{etoolbox} +\pretocmd{\section}{\clearpage}{}{} + +\providecommand{\xmpquote}[1]{#1} \ No newline at end of file diff --git a/template/ieee.csl b/template/ieee.csl new file mode 100644 index 0000000..98ed32c --- /dev/null +++ b/template/ieee.csl @@ -0,0 +1,519 @@ + +