diff --git a/README.md b/README.md index c66a55f..2fc2cd8 100644 --- a/README.md +++ b/README.md @@ -73,28 +73,69 @@ requirements are as follows: ### Setup DB -Make sure the env file contains a valid DB_URL for your setup. If you're -following this setup exactly, the value is already correct -(`DB_URL="postgresql:///cape_env_db"`). - -**_NOTE:_** You can also specify this in an environment variable name `DB_URL`. -This is intended more for automation use cases where the installation of the -package is less controllable. - This package provides the `capedb` script to handle DB upgrades, downgrades and checking of current version. All other `alembic` commands (including upgrades, downgrades and checking of current version) can be accomplished via normal `alembic` means. +**_NOTE:_** At this point we expect an empty database to exist before we apply +migrations or create tables. This could change in the future, but for now it's a +requirement. If you followed the postgres setup above you have already done +this. + +#### Alembic Config + +Usage of the `capedb` script requires an `alembic` config file be available. +This can be accomplished in a number of ways: + +- have it available in the current working directory when executing the `capedb` + script (it will be found automatically) +- specification on the command line when calling `capedb`: + `capedb [-c | --config ]` +- specifying the location in the `ALEMBIC_CONFIG` environment variable + +**_NOTE:_** However the `alembic.ini` is specified, the value of +`script_location` needs to point to the migrations directory of this package. +This is most likely `/cape_cod_db/migrations` if you +are using an installed version of this package. The +`/cape_cod_db` directory also contains an `alembic.ini` +that can be used if defaults are fine, though you will want to specify the +database url as detailed below if you use the builtin config file. + +#### Database URL + +Though the database url is often defined in the `alembic` config, if you also +need to specify it separately you also have a few options: + +- specification on the command line when calling `capedb`: + `capedb [-x db_url=]` +- specifying the location in the `DB_URL` environment variable + +#### capedb Script Usage + Run the alembic migrations on the empty (or previously upgraded) db to get up to date `capedb upgrade head`. _NOTE:_ `head` can be replaced with another revision identifier to go to a specific version. -Downgrades can be performed via `capedb upgrade `. +Downgrades can be performed via `capedb downgrade `. + +The current version can be determined with `capedb current [--verbose]`. + +#### capedb-app script + +This is probably not anything you want to use. Probably. + +This is the more traditional `app.py` like mechanism to create database tables. +This is not compatible with our provided migrations either. If you wish to +create an empty database using only the most schema in this version of this +package without any migration information, this is your boo. But you would have +to maintain your own migrations at that point. If you are maintaining migrations +ever. -The current version can be determined with `capedb current [--verbose]` +This script requires the `DB_URL` environment variable be set and does not work +with alembic at all, so the alembic config is not needed. -### play with it +### Play With The DB From repo root in the python repl of your fancy (that has all the dependencies installed). @@ -139,3 +180,50 @@ with Session(db.engine) as session: for u in res.all(): print(u) ``` + +If you prefer `psql` (**_NOTE:_** this assumes you have all the perms on the +db): + +```bash +12:12 $ psql cape_env_db + + +psql (18.3) +Type "help" for help. + +cape_env_db=> \d+ + List of relations + Schema | Name | Type | Owner | Persistence | Access method | Size | Description +--------+-----------------+----------+-------+-------------+---------------+------------+------------- + public | alembic_version | table | xxxx | permanent | heap | 8192 bytes | + public | user | table | xxxx | permanent | heap | 16 kB | + public | user_id_seq | sequence | xxxx | permanent | | 8192 bytes | + +cape_env_db=> \d user + Table "public.user" + Column | Type | Collation | Nullable | Default +-------------+-----------------------------+-----------+----------+---------------------------------- + created_at | timestamp without time zone | | not null | + last_edited | timestamp without time zone | | not null | + id | integer | | not null | nextval('user_id_seq'::regclass) + first_name | character varying | | not null | + last_name | character varying | | not null | + email | character varying | | not null | + +cape_env_db=> select * from alembic_version; + version_num +-------------- + 6001985fea71 +(1 row) + +# NOTE that if you manipulate records via sql, the created_at and last_edited +# **ARE NOT HANDLED FOR YOU** + +cape_env_db=> insert into public.user (created_at, last_edited, first_name, last_name, email) values (now(), now(), 'First', 'Last', 'fl@fakeemail.test'); +INSERT 0 1 +cape_env_db=> select * from public.user; + created_at | last_edited | id | first_name | last_name | email +----------------------------+----------------------------+----+------------+-----------+------------------- + 2026-03-05 13:55:21.688756 | 2026-03-05 13:55:21.688756 | 2 | First | Last | fl@fakeemail.test +(1 row) +``` diff --git a/app.py b/app.py deleted file mode 100644 index b7cb347..0000000 --- a/app.py +++ /dev/null @@ -1,15 +0,0 @@ -from cape_cod_db.database import create_db_and_tables -from cape_cod_db.models import User - -# NOTE: if we need to create fixtures in code, do that here. add functions for -# each fixture set and call them in main after creating the tables. if we -# end up providing dumps of fixtures instead, we don't need to add code, -# just import the dumps (probably) - - -def main(): - create_db_and_tables() - - -if __name__ == "__main__": - main() diff --git a/alembic.ini b/cape_cod_db/alembic.ini similarity index 99% rename from alembic.ini rename to cape_cod_db/alembic.ini index 8fce0f8..d33d60e 100644 --- a/alembic.ini +++ b/cape_cod_db/alembic.ini @@ -5,7 +5,7 @@ # this is typically a path given in POSIX (e.g. forward slashes) # format, relative to the token %(here)s which refers to the location of this # ini file -script_location = %(here)s/alembic +script_location = %(here)s/migrations # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s # Uncomment the line below if you want the files to be prepended with date and time diff --git a/cape_cod_db/cli/__init__.py b/cape_cod_db/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cape_cod_db/cli/app.py b/cape_cod_db/cli/app.py new file mode 100644 index 0000000..02ad76e --- /dev/null +++ b/cape_cod_db/cli/app.py @@ -0,0 +1,15 @@ +from cape_cod_db.database import create_tables +from cape_cod_db.models import User + +# WARNING: Usage of this script only applies the most recent schema, with 0 +# respect to migrations and without including any migration tables or +# history. If migrations are desired (probable) use `capedb.py` +# instead. + + +def main(): + create_tables() + + +if __name__ == "__main__": + main() diff --git a/capedb.py b/cape_cod_db/cli/capedb.py similarity index 100% rename from capedb.py rename to cape_cod_db/cli/capedb.py diff --git a/cape_cod_db/database.py b/cape_cod_db/database.py index 721034f..a2c4b50 100644 --- a/cape_cod_db/database.py +++ b/cape_cod_db/database.py @@ -1,25 +1,46 @@ import logging -from dotenv import dotenv_values from sqlmodel import SQLModel, create_engine -config = dotenv_values(".env") +db_url = None -logging.basicConfig() -logger = logging.getLogger("sqlalchemy.engine") -log_level = config.get("LOG_LEVEL", logging.INFO) -assert log_level is not None, "Log level should not be None" -logger.setLevel(log_level) +try: + from .migrations.env import config + logger = logging.getLogger("alembic.env") -db_url = config.get("DB_URL", None) + # this works for the case where we're doing alembic things. When we're + # doing orm things (e.g. in an api lambda) we need another source to check + # some other source for the DB URL as the config object doesn't exit in + # env.py. alembic doesn't play when we're just doing orm things. + db_url = config.get_main_option("sqlalchemy.url") +except AttributeError as ae: + logging.basicConfig() + logger = logging.getLogger(__file__) + logger.warning( + "Not running with a valid alembic config. Checking for DB_URL " + "environment variable" + ) + + import os + + db_url = os.getenv("DB_URL") if db_url is None: - logger.error("DB_URL is not configured") + logger.error( + "DB_URL is not configured. This needs to be in the alembic config " + "file or an environment variable named `DB_URL`" + ) exit(1) +logger.info(f"Configured for database: {db_url}") + engine = create_engine(db_url) -def create_db_and_tables(): +def create_tables(): + """Create the tables on the DB pointed to by `engine`. + + At this point we expect the empty database to exist when calling this. + """ SQLModel.metadata.create_all(engine) diff --git a/alembic/README b/cape_cod_db/migrations/README similarity index 100% rename from alembic/README rename to cape_cod_db/migrations/README diff --git a/cape_cod_db/migrations/__init__.py b/cape_cod_db/migrations/__init__.py new file mode 100644 index 0000000..81d65c1 --- /dev/null +++ b/cape_cod_db/migrations/__init__.py @@ -0,0 +1,4 @@ +from pathlib import Path + +# export of the default alembic config installed with this package +DEFAULT_ALEMBIC_CFGPATH = Path(__file__).parent.parent / "alembic.ini" diff --git a/alembic/env.py b/cape_cod_db/migrations/env.py similarity index 76% rename from alembic/env.py rename to cape_cod_db/migrations/env.py index 7d41c04..fbab9c9 100644 --- a/alembic/env.py +++ b/cape_cod_db/migrations/env.py @@ -1,20 +1,11 @@ +import logging import os -import sys - -# so that alembic has access to our models, we're adding the project root here -proj_root = os.path.realpath(os.path.join(os.path.dirname(__file__), "..")) -sys.path.insert(0, proj_root) - -print(sys.path) - from logging.config import fileConfig -from dotenv import dotenv_values +from alembic import context from sqlalchemy import engine_from_config, pool from sqlmodel import SQLModel -from alembic import context - # NOTE: as new table models are added, they need to be imported here. from cape_cod_db.models import User @@ -27,6 +18,8 @@ if config.config_file_name is not None: fileConfig(config.config_file_name) +logger = logging.getLogger("alembic.env") + # add your model's MetaData object here # for 'autogenerate' support # from myapp import mymodel @@ -34,36 +27,29 @@ target_metadata = SQLModel.metadata -# cape_cod_db has its own config file and which contains database url. -# Additionally for migrations we're running from an ephemeral virtualenv that -# may not have a good .env file available. -# we're going to ignore what's set in the alembic ini then check the following -# in decreasing order of precedence: -# - an environment variable named "DB_URL" -# - the project .env file for a key of "DB_URL" -# We'll consider one of those to be required and will error out if not found. +# we have 3 ways to specify the database url (higher in list is higher +# precedence): +# - command line parameter `db_url` +# - env var `DB_URL` +# - alembic config file value `sqlalchemy.url` -db_url = os.getenv("DB_URL") +cli_args = context.get_x_argument(as_dictionary=True) -if db_url is None: - # TODO: we also specify a log level in that config. we *could* use that here, - # but for now we're going to respect what's in the alembic ini since the - # logging here is for a different purpose than the logging in the db app's - # setup - proj_config = dotenv_values(os.path.join(proj_root, ".env")) - db_url = proj_config.get("DB_URL") +# first try a cli arg +db_url = cli_args.get("db_url", None) +# then an env var if db_url is None: - print( - "No DB_URL configured in environment variable or project level .env " - "file. Cannot continue." - ) - # if we don't have this we have problems. so we are bailing - sys.exit(1) - + db_url = os.getenv("DB_URL") -config.set_main_option("sqlalchemy.url", db_url) +# we've already got the file config, so if db_url is not None by here, overwrite +# the file config value +if db_url is not None: + config.set_main_option("sqlalchemy.url", db_url) +logging.info( + f"Configured for database: {config.get_main_option('sqlalchemy.url')}" +) # other values from the config, defined by the needs of env.py, # can be acquired: diff --git a/alembic/script.py.mako b/cape_cod_db/migrations/script.py.mako similarity index 100% rename from alembic/script.py.mako rename to cape_cod_db/migrations/script.py.mako diff --git a/alembic/versions/6001985fea71_add_email_to_user_table.py b/cape_cod_db/migrations/versions/6001985fea71_add_email_to_user_table.py similarity index 100% rename from alembic/versions/6001985fea71_add_email_to_user_table.py rename to cape_cod_db/migrations/versions/6001985fea71_add_email_to_user_table.py diff --git a/alembic/versions/eecb735a6c3b_create_user_table.py b/cape_cod_db/migrations/versions/eecb735a6c3b_create_user_table.py similarity index 100% rename from alembic/versions/eecb735a6c3b_create_user_table.py rename to cape_cod_db/migrations/versions/eecb735a6c3b_create_user_table.py diff --git a/poetry.lock b/poetry.lock index cbfedbf..8bd0a9f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6,7 +6,7 @@ version = "1.18.4" description = "A database migration tool for SQLAlchemy." optional = false python-versions = ">=3.10" -groups = ["dev"] +groups = ["main"] files = [ {file = "alembic-1.18.4-py3-none-any.whl", hash = "sha256:a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a"}, {file = "alembic-1.18.4.tar.gz", hash = "sha256:cb6e1fd84b6174ab8dbb2329f86d631ba9559dd78df550b57804d607672cedbc"}, @@ -27,7 +27,7 @@ version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main"] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -39,7 +39,7 @@ version = "3.3.2" description = "Lightweight in-process concurrent programming" optional = false python-versions = ">=3.10" -groups = ["dev"] +groups = ["main"] markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\"" files = [ {file = "greenlet-3.3.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9bc885b89709d901859cf95179ec9f6bb67a3d2bb1f0e88456461bd4b7f8fd0d"}, @@ -107,7 +107,7 @@ version = "1.3.10" description = "A super-fast templating language that borrows the best ideas from the existing templating languages." optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main"] files = [ {file = "mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59"}, {file = "mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28"}, @@ -127,7 +127,7 @@ version = "3.0.3" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main"] files = [ {file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"}, {file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"}, @@ -226,7 +226,7 @@ version = "2.9.11" description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main"] files = [ {file = "psycopg2-binary-2.9.11.tar.gz", hash = "sha256:b6aed9e096bf63f9e75edf2581aa9a7e7186d97ab5c177aa6c87797cd591236c"}, {file = "psycopg2_binary-2.9.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d6fe6b47d0b42ce1c9f1fa3e35bb365011ca22e39db37074458f27921dca40f2"}, @@ -303,7 +303,7 @@ version = "2.12.5" description = "Data validation using Python type hints" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main"] files = [ {file = "pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d"}, {file = "pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49"}, @@ -325,7 +325,7 @@ version = "2.41.5" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main"] files = [ {file = "pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146"}, {file = "pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2"}, @@ -453,28 +453,13 @@ files = [ [package.dependencies] typing-extensions = ">=4.14.1" -[[package]] -name = "python-dotenv" -version = "1.2.2" -description = "Read key-value pairs from a .env file and set them as environment variables" -optional = false -python-versions = ">=3.10" -groups = ["dev"] -files = [ - {file = "python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a"}, - {file = "python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3"}, -] - -[package.extras] -cli = ["click (>=5.0)"] - [[package]] name = "sqlalchemy" version = "2.0.48" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" -groups = ["dev"] +groups = ["main"] files = [ {file = "sqlalchemy-2.0.48-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7001dc9d5f6bb4deb756d5928eaefe1930f6f4179da3924cbd95ee0e9f4dce89"}, {file = "sqlalchemy-2.0.48-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1a89ce07ad2d4b8cfc30bd5889ec40613e028ed80ef47da7d9dd2ce969ad30e0"}, @@ -576,7 +561,7 @@ version = "0.0.37" description = "SQLModel, SQL databases in Python, designed for simplicity, compatibility, and robustness." optional = false python-versions = ">=3.10" -groups = ["dev"] +groups = ["main"] files = [ {file = "sqlmodel-0.0.37-py3-none-any.whl", hash = "sha256:2137a4045ef3fd66a917a7717ada959a1ceb3630d95e1f6aaab39dd2c0aef278"}, {file = "sqlmodel-0.0.37.tar.gz", hash = "sha256:d2c19327175794faf50b1ee31cc966764f55b1dedefc046450bc5741a3d68352"}, @@ -592,7 +577,7 @@ version = "2.4.0" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main"] markers = "python_version == \"3.10\"" files = [ {file = "tomli-2.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b5ef256a3fd497d4973c11bf142e9ed78b150d36f5773f1ca6088c230ffc5867"}, @@ -650,7 +635,7 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, @@ -662,7 +647,7 @@ version = "0.4.2" description = "Runtime typing introspection tools" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main"] files = [ {file = "typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7"}, {file = "typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464"}, @@ -674,4 +659,4 @@ typing-extensions = ">=4.12.0" [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "e457989b445bcc50a73da1069ac767412ccb52ee959b86b6983a7a710e6af16b" +content-hash = "88c72aa5a8140b3d2a40f0fb6307920af40be74748d9aba4183e5027dcbd725b" diff --git a/pyproject.toml b/pyproject.toml index 5267f1f..622496d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,31 +6,28 @@ authors = ["L. Drew Pihera "] license = "Apache-2.0" readme = "README.md" packages = [{ include = "cape_cod_db" }] -include = ["app.py", "capedb.py", "alembic", "alembic.ini"] [tool.poetry.dependencies] python = "^3.10" - -[tool.poetry.group.dev.dependencies] alembic = "^1.18.4" -python-dotenv = "^1.2.1" psycopg2-binary = "^2.9.11" sqlmodel = "^0.0.37" [tool.poetry.scripts] -capedb = "capedb:main" +capedb = "cape_cod_db.cli.capedb:main" +capedb-app = "cape_cod_db.cli.app:main" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.black] -force-exclude = "alembic" +force-exclude = "cape_cod_db/migrations" line-length = 80 [tool.isort] profile = "black" -extend_skip = ["alembic"] +extend_skip = ["cape_cod_db/migrations"] line_length = 80 diff --git a/pyrightconfig.json b/pyrightconfig.json index 78030fa..3505172 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,5 +1,5 @@ { "autoImportCompletions": true, "typeCheckingMode": "basic", - "ignore": ["alembic/**"] + "ignore": ["cape_cod_db/migrations/**"] }