Q
02/04/2023, 10:20 AMsqlalchemy.exc.OperationalError: (sqlite3.OperationalError) duplicate column name: has_data
(see thread). Seems like the culpable migration is in a commit merged last week.
Downgraded to 2.7.10, got alembic.util.exc.CommandError: Can't locate revision identified by 'f92143d30c24'
.
Managed to solve by running prefect orion database downgrade -r bb38729c471a
from 2.7.11 and downgrading to 2.7.10, worked.
Tried to reproduce by upgrading to 2.7.11 and starting the server => same exception, same solution.
Can't say if this was the original reason for crashlooping or something else left database in a weird state.
Orion is running on k8s and using sqlite.___ ___ ___ ___ ___ ___ _____ ___ ___ ___ ___ _ _
| _ \ _ \ __| __| __/ __|_ _| / _ \| _ \_ _/ _ \| \| |
| _/ / _|| _|| _| (__ | | | (_) | /| | (_) | .` |
|_| |_|_\___|_| |___\___| |_| \___/|_|_\___\___/|_|\_|
Configure Prefect to communicate with the server with:
prefect config set PREFECT_API_URL=<http://0.0.0.0:4200/api>
View the API reference documentation at <http://0.0.0.0:4200/docs>
Check out the dashboard at <http://0.0.0.0:4200>
Traceback (most recent call last):
File "/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1900, in _execute_context
self.dialect.do_execute(
File "/venv/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 736, in do_execute
cursor.execute(statement, parameters)
File "/venv/lib/python3.9/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 108, in execute
self._adapt_connection._handle_exception(error)
File "/venv/lib/python3.9/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 236, in _handle_exception
raise error
File "/venv/lib/python3.9/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 90, in execute
self.await_(_cursor.execute(operation, parameters))
File "/venv/lib/python3.9/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 68, in await_only
return current.driver.switch(awaitable)
File "/venv/lib/python3.9/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 121, in greenlet_spawn
value = await result
File "/venv/lib/python3.9/site-packages/aiosqlite/cursor.py", line 37, in execute
await self._execute(self._cursor.execute, sql, parameters)
File "/venv/lib/python3.9/site-packages/aiosqlite/cursor.py", line 31, in _execute
return await self._conn._execute(fn, *args, **kwargs)
File "/venv/lib/python3.9/site-packages/aiosqlite/core.py", line 137, in _execute
return await future
File "/venv/lib/python3.9/site-packages/aiosqlite/core.py", line 110, in run
result = function()
sqlite3.OperationalError: duplicate column name: has_data
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/venv/lib/python3.9/site-packages/starlette/routing.py", line 671, in lifespan
async with self.lifespan_context(app):
File "/venv/lib/python3.9/site-packages/starlette/routing.py", line 566, in __aenter__
await self._router.startup()
File "/venv/lib/python3.9/site-packages/starlette/routing.py", line 648, in startup
await handler()
File "/venv/lib/python3.9/site-packages/prefect/orion/api/server.py", line 348, in run_migrations
await db.create_db()
File "/venv/lib/python3.9/site-packages/prefect/orion/database/interface.py", line 55, in create_db
await self.run_migrations_upgrade()
File "/venv/lib/python3.9/site-packages/prefect/orion/database/interface.py", line 63, in run_migrations_upgrade
await run_sync_in_worker_thread(alembic_upgrade)
File "/venv/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 91, in run_sync_in_worker_thread
return await anyio.to_thread.run_sync(
File "/venv/lib/python3.9/site-packages/anyio/to_thread.py", line 31, in run_sync
return await get_asynclib().run_sync_in_worker_thread(
File "/venv/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 937, in run_sync_in_worker_thread
return await future
File "/venv/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 867, in run
result = context.run(func, *args)
File "/venv/lib/python3.9/site-packages/prefect/orion/database/alembic_commands.py", line 24, in wrapper
return fn(*args, **kwargs)
File "/venv/lib/python3.9/site-packages/prefect/orion/database/alembic_commands.py", line 53, in alembic_upgrade
alembic.command.upgrade(alembic_config(), revision, sql=dry_run)
File "/venv/lib/python3.9/site-packages/alembic/command.py", line 378, in upgrade
script.run_env()
File "/venv/lib/python3.9/site-packages/alembic/script/base.py", line 569, in run_env
util.load_python_file(self.dir, "env.py")
File "/venv/lib/python3.9/site-packages/alembic/util/pyfiles.py", line 94, in load_python_file
module = load_module_py(module_id, path)
File "/venv/lib/python3.9/site-packages/alembic/util/pyfiles.py", line 110, in load_module_py
spec.loader.exec_module(module) # type: ignore
File "<frozen importlib._bootstrap_external>", line 850, in exec_module
File "<frozen importlib._bootstrap>", line 228, in _call_with_frames_removed
File "/venv/lib/python3.9/site-packages/prefect/orion/database/migrations/env.py", line 147, in <module>
apply_migrations()
File "/venv/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 226, in coroutine_wrapper
return run_async_from_worker_thread(async_fn, *args, **kwargs)
File "/venv/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 177, in run_async_from_worker_thread
return anyio.from_thread.run(call)
File "/venv/lib/python3.9/site-packages/anyio/from_thread.py", line 49, in run
return asynclib.run_async_from_thread(func, *args)
File "/venv/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 970, in run_async_from_thread
return f.result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 445, in result
return self.__get_result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 390, in __get_result
raise self._exception
File "/venv/lib/python3.9/site-packages/prefect/orion/database/migrations/env.py", line 141, in apply_migrations
await connection.run_sync(do_run_migrations)
File "/venv/lib/python3.9/site-packages/sqlalchemy/ext/asyncio/engine.py", line 548, in run_sync
return await greenlet_spawn(fn, conn, *arg, **kw)
File "/venv/lib/python3.9/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 128, in greenlet_spawn
result = context.switch(value)
File "/venv/lib/python3.9/site-packages/prefect/orion/database/migrations/env.py", line 129, in do_run_migrations
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/venv/lib/python3.9/site-packages/alembic/runtime/environment.py", line 867, in run_migrations
self.get_context().run_migrations(**kw)
File "/venv/lib/python3.9/site-packages/alembic/runtime/migration.py", line 624, in run_migrations
step.migration_fn(**kw)
File "/venv/lib/python3.9/site-packages/prefect/orion/database/migrations/versions/sqlite/2023_01_12_000043_f92143d30c25_create_migration_index.py", line 23, in upgrade
batch_op.create_index(
File "/usr/local/lib/python3.9/contextlib.py", line 126, in __exit__
next(self.gen)
File "/venv/lib/python3.9/site-packages/alembic/operations/base.py", line 383, in batch_alter_table
impl.flush()
File "/venv/lib/python3.9/site-packages/alembic/operations/batch.py", line 111, in flush
fn(*arg, **kw)
File "/venv/lib/python3.9/site-packages/alembic/ddl/impl.py", line 322, in add_column
self._exec(base.AddColumn(table_name, column, schema=schema))
File "/venv/lib/python3.9/site-packages/alembic/ddl/impl.py", line 193, in _exec
return conn.execute( # type: ignore[call-overload]
File "/venv/lib/python3.9/site-packages/sqlalchemy/future/engine.py", line 280, in execute
return self._execute_20(
File "/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1705, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/venv/lib/python3.9/site-packages/sqlalchemy/sql/ddl.py", line 80, in _execute_on_connection
return connection._execute_ddl(
File "/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1472, in _execute_ddl
ret = self._execute_context(
File "/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1943, in _execute_context
self._handle_dbapi_exception(
File "/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2124, in _handle_dbapi_exception
util.raise_(
File "/venv/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 211, in raise_
raise exception
File "/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1900, in _execute_context
self.dialect.do_execute(
File "/venv/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 736, in do_execute
cursor.execute(statement, parameters)
File "/venv/lib/python3.9/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 108, in execute
self._adapt_connection._handle_exception(error)
File "/venv/lib/python3.9/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 236, in _handle_exception
raise error
File "/venv/lib/python3.9/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 90, in execute
self.await_(_cursor.execute(operation, parameters))
File "/venv/lib/python3.9/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 68, in await_only
return current.driver.switch(awaitable)
File "/venv/lib/python3.9/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 121, in greenlet_spawn
value = await result
File "/venv/lib/python3.9/site-packages/aiosqlite/cursor.py", line 37, in execute
await self._execute(self._cursor.execute, sql, parameters)
File "/venv/lib/python3.9/site-packages/aiosqlite/cursor.py", line 31, in _execute
return await self._conn._execute(fn, *args, **kwargs)
File "/venv/lib/python3.9/site-packages/aiosqlite/core.py", line 137, in _execute
return await future
File "/venv/lib/python3.9/site-packages/aiosqlite/core.py", line 110, in run
result = function()
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) duplicate column name: has_data
[SQL: ALTER TABLE flow_run_state ADD COLUMN has_data BOOLEAN]
(Background on this error at: <https://sqlalche.me/e/14/e3q8>)
Application startup failed. Exiting.
Orion stopped!