Merge pull request #280 from tracim/dev/233/files_on_disk

doc/migration.md

+# Performing migrations #
+
+## Introduction ##
+
+This document is intended to developers.
+
+Migrations on `Tracim` lays on [`gearbox migrate`](http://turbogears.readthedocs.io/en/tg2.3.7/turbogears/migrations.html), which in turn lays on [`alembic`](http://alembic.zzzcomputing.com/en/latest/index.html) which is the migration tool dedicated to `SQLAlchemy`.
+
+In order to use the `gearbox migrate [...]` commands, change your current directory to be `tracim/` from the root of the project, also usually named `tracim/` :
+
+    (tg2env) user@host:~/tracim$ cd tracim/
+    (tg2env) user@host:~/tracim/tracim$
+
+## Migration howto - Overview ##
+
+### Upgrading schema ###
+
+    gearbox migrate upgrade
+
+### Downgrading schema ###
+
+    gearbox migrate downgrade
+
+## Migration howto - Advanced (for developers) ##
+
+### Retrieving schema current version ###
+
+    gearbox migrate db_version
+
+### Creating new schema migration ###
+
+This creates a new python migration file in `tracim/migration/versions/` ending by `migration_label.py`:
+
+    gearbox migrate create 'migration label'

tracim/migration/versions/69fb10c3d6f0_files_on_disk.py

-"""files on disk
+"""files on disk.
 
 Revision ID: 69fb10c3d6f0
 Revises: c1cea4bbae16
 down_revision = 'c1cea4bbae16'
 
 
+# INFO - A.P - 2017-07-20 - alembic batch migrations
+# http://alembic.zzzcomputing.com/en/latest/batch.html
+# This migration uses alembic batch mode, a workaround allowing to enforce
+# ALTER statement with SQLite while maintaining the traditional behavior of
+# the commented lines on other RDBMS.
+
+
 def upgrade():
-    op.add_column('content_revisions',
-                  sa.Column('depot_file',
-                            UploadedFileField))
+    """Adds the depot file in revision."""
+    # op.add_column('content_revisions',
+    #               sa.Column('depot_file',
+    #                         UploadedFileField))
+    with op.batch_alter_table('content_revisions') as batch_op:
+        batch_op.add_column(sa.Column('depot_file', UploadedFileField))
 
 
 def downgrade():
-    op.drop_column('content_revisions', 'depot_file')
+    """Drops the depot file in revision."""
+    # op.drop_column('content_revisions', 'depot_file')
+    with op.batch_alter_table('content_revisions') as batch_op:
+        batch_op.drop_column('depot_file')

tracim/migration/versions/913efdf409e5_all_files_also_on_disk.py

+"""all files also on disk.
+
+Revision ID: 913efdf409e5
+Revises: 69fb10c3d6f0
+Create Date: 2017-07-12 15:44:20.568447
+
+"""
+
+import shutil
+
+from alembic import op
+from depot.fields.sqlalchemy import UploadedFileField
+from depot.fields.upload import UploadedFile
+from depot.io.utils import FileIntent
+from depot.manager import DepotManager
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '913efdf409e5'
+down_revision = '69fb10c3d6f0'
+
+
+revision_helper = sa.Table(
+    'content_revisions',
+    sa.MetaData(),
+    sa.Column('revision_id', sa.Integer, primary_key=True),
+    sa.Column('label', sa.String(1024), nullable=False),
+    sa.Column('file_extension', sa.String(255), nullable=False),
+    sa.Column('file_mimetype', sa.String(255), nullable=False),
+    sa.Column('file_content', sa.LargeBinary),
+    sa.Column('depot_file', UploadedFileField, nullable=True),
+    sa.Column('type', sa.String(32), nullable=False),
+)
+
+
+def delete_files_on_disk(connection: sa.engine.Connection):
+    """Deletes files from disk and their references in database."""
+    delete_query = revision_helper.update() \
+        .where(revision_helper.c.type == 'file') \
+        .where(revision_helper.c.depot_file.isnot(None)) \
+        .values(depot_file=None)
+    connection.execute(delete_query)
+    shutil.rmtree('depot/', ignore_errors=True)
+
+
+def upgrade():
+    """
+    Sets all depot files for file typed revisions.
+
+    Until now, files are both in database and, for the newly created
+    ones, on disk. In order to simplify the migration, this procedure
+    will:
+    - delete the few files on disk,
+    - create all files on disk from database.
+    """
+    # Creates files depot used in this migration
+    DepotManager.configure(
+        'tracim', {'depot.storage_path': 'depot/'},
+    )
+    connection = op.get_bind()
+    delete_files_on_disk(connection=connection)
+    select_query = revision_helper.select() \
+        .where(revision_helper.c.type == 'file') \
+        .where(revision_helper.c.depot_file.is_(None))
+    files = connection.execute(select_query).fetchall()
+    for file in files:
+        file_filename = '{0}{1}'.format(
+            file.label,
+            file.file_extension,
+        )
+        depot_file_intent = FileIntent(
+            file.file_content,
+            file_filename,
+            file.file_mimetype,
+        )
+        depot_file_field = UploadedFile(depot_file_intent, 'tracim')
+        update_query = revision_helper.update() \
+            .where(revision_helper.c.revision_id == file.revision_id) \
+            .values(depot_file=depot_file_field) \
+            .return_defaults()
+        connection.execute(update_query)
+
+
+def downgrade():
+    """Resets depot file for file typed revisions."""
+    connection = op.get_bind()
+    delete_files_on_disk(connection=connection)

tracim/migration/versions/f3852e1349c4_all_files_only_on_disk.py

+"""all files only on disk
+
+Revision ID: f3852e1349c4
+Revises: 913efdf409e5
+Create Date: 2017-07-24 17:15:54.278141
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'f3852e1349c4'
+down_revision = '913efdf409e5'
+
+
+def upgrade():
+    """Drops the file content from revision."""
+    with op.batch_alter_table('content_revisions') as batch_op:
+        batch_op.drop_column('file_content')
+
+
+def downgrade():
+    """Adds the file content in revision."""
+    with op.batch_alter_table('content_revisions') as batch_op:
+        batch_op.add_column(sa.Column('file_content', sa.LargeBinary))

tracim/tracim/config/app_cfg.py

 
 def configure_depot():
     """Configure Depot."""
-    depot_storage_name = 'default'
+    depot_storage_name = 'tracim'
     depot_storage_path = CFG.get_instance().DEPOT_STORAGE_DIR
     depot_storage_settings = {'depot.storage_path': depot_storage_path}
     DepotManager.configure(
         self.DEPOT_STORAGE_DIR = tg.config.get(
             'depot_storage_dir',
         )
+        if not self.DEPOT_STORAGE_DIR:
+            raise Exception(
+                'ERROR: depot_storage_dir configuration is mandatory. '
+                'Set it before continuing.'
+            )
         self.PREVIEW_CACHE_DIR = tg.config.get(
             'preview_cache_dir',
         )

tracim/tracim/fixtures/content.py

 # -*- coding: utf-8 -*-
+from depot.io.utils import FileIntent
+
 from tracim import model
 from tracim.fixtures import Fixture
 from tracim.fixtures.users_and_groups import Test
             do_save=False,
         )
         w1f1d1_txt.file_extension = '.txt'
-        w1f1d1_txt.file_content = b'w1f1d1 content'
+        w1f1d1_txt.depot_file = FileIntent(
+            b'w1f1d1 content',
+            'w1f1d1.txt',
+            'text/plain',
+        )
         self._session.add(w1f1d1_txt)
         w1f1d2_html = content_api.create(
             content_type=ContentType.File,
             do_save=False,
         )
         w1f1d2_html.file_extension = '.html'
-        w1f1d2_html.file_content = b'<p>w1f1d2 content</p>'
+        w1f1d2_html.depot_file = FileIntent(
+            b'<p>w1f1d2 content</p>',
+            'w1f1d2.html',
+            'text/html',
+        )
         self._session.add(w1f1d2_html)
         w1f1f1 = content_api.create(
             content_type=ContentType.Folder,

tracim/tracim/lib/content.py

 from tg.i18n import ugettext as _
 
 from depot.manager import DepotManager
+from depot.io.utils import FileIntent
 
 import sqlalchemy
 from sqlalchemy.orm import aliased
         item.revision_type = ActionDescription.EDITION
         return item
 
-    def update_file_data(self, item: Content, new_filename: str, new_mimetype: str, new_file_content) -> Content:
+    def update_file_data(self, item: Content, new_filename: str, new_mimetype: str, new_content: bytes) -> Content:
         item.owner = self._user
         item.file_name = new_filename
         item.file_mimetype = new_mimetype
-        item.file_content = new_file_content
-        item.depot_file = new_file_content
+        item.depot_file = FileIntent(
+            new_content,
+            new_filename,
+            new_mimetype,
+        )
         item.revision_type = ActionDescription.REVISION
         return item
 

tracim/tracim/lib/webdav/sql_resources.py

 
 import tg
 import transaction
+import typing
 import re
 from datetime import datetime
 from time import mktime
         return "<DAVNonCollection: File (%d)>" % self.content.revision_id
 
     def getContentLength(self) -> int:
-        return len(self.content.file_content)
+        return self.content.depot_file.file.content_length
 
     def getContentType(self) -> str:
         return self.content.file_mimetype
     def getLastModified(self) -> float:
         return mktime(self.content.updated.timetuple())
 
-    def getContent(self):
+    def getContent(self) -> typing.BinaryIO:
         filestream = compat.BytesIO()
-        filestream.write(self.content.file_content)
+        filestream.write(self.content.depot_file.file.read())
         filestream.seek(0)
 
         return filestream
 
     def getContent(self):
         filestream = compat.BytesIO()
-        filestream.write(self.content_revision.file_content)
+        filestream.write(self.content_revision.depot_file.file.read())
         filestream.seek(0)
 
         return filestream
 
     def getContentLength(self):
-        return len(self.content_revision.file_content)
+        return self.content_revision.depot_file.file.content_length
 
     def getContentType(self) -> str:
         return self.content_revision.file_mimetype

tracim/tracim/model/data.py

 from sqlalchemy.types import Unicode
 from depot.fields.sqlalchemy import UploadedFileField
 from depot.fields.upload import UploadedFile
+from depot.io.utils import FileIntent
 
 from tracim.lib.utils import lazy_ugettext as l_
 from tracim.lib.exception import ContentRevisionUpdateError
         server_default='',
     )
     file_mimetype = Column(Unicode(255),  unique=False, nullable=False, default='')
-    # TODO - A.P - 2017-07-03 - future removal planned
-    # file_content is to be replaced by depot_file, for now both coexist as
-    # this:
-    # - file_content data is still setted
-    # - newly created revision also gets depot_file data setted
-    # - access to the file of a revision from depot_file exclusively
-    # Here is the tasks workflow of the DB to OnDisk Switch :
-    # - Add depot_file "prototype style"
-    #   https://github.com/tracim/tracim/issues/233 - DONE
-    # - Integrate preview generator feature "prototype style"
-    #   https://github.com/tracim/tracim/issues/232 - DONE
-    # - Write migrations
-    #   https://github.com/tracim/tracim/issues/245
-    #   https://github.com/tracim/tracim/issues/246
-    # - Stabilize preview generator integration
-    #   includes dropping DB file content
-    #   https://github.com/tracim/tracim/issues/249
-    file_content = deferred(Column(LargeBinary(), unique=False, nullable=True))
     # INFO - A.P - 2017-07-03 - Depot Doc
     # http://depot.readthedocs.io/en/latest/#attaching-files-to-models
     # http://depot.readthedocs.io/en/latest/api.html#module-depot.fields
         'content_id',
         'created',
         'description',
-        'file_content',
         'file_mimetype',
         'file_extension',
         'is_archived',
             setattr(new_rev, column_name, column_value)
 
         new_rev.updated = datetime.utcnow()
-        # TODO APY tweaks here depot_file
-        # import pudb; pu.db
-        # new_rev.depot_file = DepotManager.get().get(revision.depot_file)
-        new_rev.depot_file = revision.file_content
+        if revision.depot_file:
+            new_rev.depot_file = FileIntent(
+                revision.depot_file.file.read(),
+                revision.file_name,
+                revision.file_mimetype,
+            )
 
         return new_rev
 
         return ContentRevisionRO.file_mimetype
 
     @hybrid_property
-    def file_content(self):
-        return self.revision.file_content
-
-    @file_content.setter
-    def file_content(self, value):
-        self.revision.file_content = value
-
-    @file_content.expression
-    def file_content(cls) -> InstrumentedAttribute:
-        return ContentRevisionRO.file_content
-
-    @hybrid_property
     def _properties(self) -> str:
         return self.revision.properties
 

tracim/tracim/tests/library/test_content_api.py

                                                           updated.owner_id))
         eq_('this_is_a_page.html', updated.file_name)
         eq_('text/html', updated.file_mimetype)
-        eq_(b'<html>hello world</html>', updated.file_content)
+        eq_(b'<html>hello world</html>', updated.depot_file.file.read())
         eq_(ActionDescription.REVISION, updated.revision_type)
 
     def test_archive_unarchive(self):

tracim/tracim/tests/library/test_webdav.py

         ))
         eq_(
             b'hello\n',
-            result.content.file_content,
+            result.content.depot_file.file.read(),
             msg='fiel content should be "hello\n" but it is {0}'.format(
-                result.content.file_content
+                result.content.depot_file.file.read()
             )
         )
 
         ))
         eq_(
             b'hello\n',
-            result.content.file_content,
+            result.content.depot_file.file.read(),
             msg='fiel content should be "hello\n" but it is {0}'.format(
-                result.content.file_content
+                result.content.depot_file.file.read()
             )
         )
 

tracim/tracim/tests/models/test_content_revision.py

 class TestContentRevision(BaseTest, TestStandard):
 
     def _new_from(self, revision):
-        excluded_columns = ('revision_id', '_sa_instance_state')
+        excluded_columns = ('revision_id', '_sa_instance_state', 'depot_file')
         revision_columns = [attr.key for attr in inspect(revision).attrs if not attr.key in excluded_columns]
         new_revision = ContentRevisionRO()