Clean up

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Next
 * Add Content Security Policy directive form-action
 * Remember "Encrypt PDF" setting in browser's local storage
+* Collaborative editing in notes
 
 
 ## v2024.19 - 2024-03-05

api/generate_notice.sh

@@ -4,6 +4,7 @@ set -e
 
 allow_only="MIT"
 allow_only="$allow_only;MIT License"
+allow_only="$allow_only;BSD"
 allow_only="$allow_only;BSD License"
 allow_only="$allow_only;Apache Software License"
 allow_only="$allow_only;GNU Library or Lesser General Public License (LGPL)"
@@ -14,6 +15,7 @@ allow_only="$allow_only;Mozilla Public License 1.1 (MPL 1.1)"
 allow_only="$allow_only;Mozilla Public License 2.0 (MPL 2.0)"
 allow_only="$allow_only;Historical Permission Notice and Disclaimer (HPND)"
 allow_only="$allow_only;Python Software Foundation License"
+allow_only="$allow_only;Zope Public License"
 
 ignore="webencodings"
 ignore="$ignore pyphen"

api/src/reportcreator_api/conf/urls.py

@@ -18,7 +18,7 @@ from reportcreator_api.pentests.views import \
     ArchivedProjectViewSet, ArchivedProjectKeyPartViewSet, UserPublicKeyViewSet
 from reportcreator_api.users.views import APITokenViewSet, PentestUserViewSet, MFAMethodViewSet, AuthViewSet, AuthIdentityViewSet
 from reportcreator_api.notifications.views import NotificationViewSet
-from reportcreator_api.pentests.consumers import ProjectNotesConsumer, UserNotesConsumer, DemoConsumer
+from reportcreator_api.pentests.consumers import ProjectNotesConsumer, UserNotesConsumer
 
 
 router = DefaultRouter()
@@ -93,7 +93,6 @@ urlpatterns = [
 websocket_urlpatterns = [
     path('ws/pentestprojects/<uuid:project_id>/notes/', ProjectNotesConsumer.as_asgi(), name='projectnotebookpage-ws'),
     path('ws/pentestusers/<str:pentestuser_pk>/notes/', UserNotesConsumer.as_asgi(), name='usernotebookpage-ws'),
-    path('ws/demo/', DemoConsumer.as_asgi(), name='demo-ws'),
 ]
 
 

api/src/reportcreator_api/pentests/consumers.py

@@ -190,7 +190,7 @@ class NotesConsumerBase(WebsocketConsumerBase):
             .select_for_update(of=['self'], no_key=True) \
             .first()
         if not note:
-            raise ValidationError('Invalid path')
+            return None, None
         return note, path_parts[2]
 
     @database_sync_to_async
@@ -199,6 +199,8 @@ class NotesConsumerBase(WebsocketConsumerBase):
         # Validate path and get note
         valid_paths = {k for k, f in self.get_serializer().fields.items() if not f.read_only} - {'title', 'text'}
         note, key = self.get_note_for_update(path=content.get('path'), valid_paths=valid_paths)
+        if not note:
+            return None
 
         # Update in DB
         serializer = self.get_serializer(instance=note, data={key: content.get('value')}, partial=True)
@@ -225,12 +227,10 @@ class NotesConsumerBase(WebsocketConsumerBase):
         if not content.get('updates', []):
             raise ValidationError('No updates')
         note, key = self.get_note_for_update(path=content.get('path'), valid_paths=['title', 'text'])
+        if not note:
+            return None
 
-        version = content['version']
-        # TODO: prevent updates for versions that are too old
-        # * check if version is too old and if there are updates in between
-        # * simple timestamp comparison is not enough, because when there were no updates in between, the version is still valid
-
+        # Rebase updates
         over_updates = CollabEvent.objects \
             .filter(related_id=self.related_id) \
             .filter(path=content['path']) \
@@ -244,7 +244,6 @@ class NotesConsumerBase(WebsocketConsumerBase):
                 for u in e.data.get('updates', [])] for e in over_updates])),
         )
 
-        log.info(f'Rebased updates: {updates=}')
         if not updates:
             return None
 
@@ -252,7 +251,6 @@ class NotesConsumerBase(WebsocketConsumerBase):
         changes = updates[0].changes
         for u in updates[1:]:
             changes = changes.compose(u.changes)
-        log.info(f'Applying changes: {changes=} to "{json.dumps(note.text)}"')
         setattr(note, key, changes.apply(getattr(note, key) or ''))
         with collab_context(prevent_events=True):
             note.save()
@@ -359,82 +357,3 @@ def send_collab_event_user(event: CollabEvent):
         'path': event.path,
     })
 
-
-class DemoConsumer(JsonWebsocketConsumer):
-    def connect(self):
-        self.accept()
-        self.send_json({'message': 'Hello World'})
-
-    def receive_json(self, data):
-        self.send_json(data)
-
-
-
-# TODO: concurrent editing
-# * [x] server config
-#   * [x] uvicorn
-#   * [x] asgi+channels
-#   * [x] channels layer: 
-#     * [x] add postgres layer for self-hosted
-#     * [x] has a max message size => circumvent via separate DB model and pass only model ID
-#     * [x] does not work with pgbouncer => affects SysReptor cloud, alternative: redis or rabbitmq?
-# * [x] reverse proxy config
-#   * [x] caddy => no config update required
-#   * [x] nginx => requires config update
-# * [x] SysReptor cloud
-#   * [x] redis or rabbitmq instead of postgres channels layer
-#   * [x] nginx config update: allow websockets
-# * [x] models
-#   * [x] NotebookPage: remove lock_info
-#   * [x] CollabEvent: id, parent_id, version, path, data
-#   * [x] delete old CollabEvent in periodic_task (e.g. once per hour, older than 2 hours)
-# * [x] consumers
-#   * [x] project notes
-#   * [x] user notes
-# * [ ] sync with DB
-#   * [x] websocket update => django ORM
-#   * [ ] post_save signal => websocket update message
-#     * [x] create
-#     * [x] delete
-#     * [x] sort
-#     * [x] update_key
-#     * [ ] update_text => get text diff and convert to ChangeSet
-# * [ ] frontend
-#   * [x] codemirror collab: emit/receive updates
-#   * [x] emit/receive update.key messages
-#   * [x] integrate to pinia store / state management
-#   * [x] integrate codemirror collab with server
-#   * [x] debounce/throttle updates in frontend: e.g. max. 1 per second
-#   * [x] on websocket disconnect: show warning and reconnect button
-#   * [x] fallback to API (read-only) if permission denied of websocket error
-#   * [x] update EditToolbar usage
-#   * [x] handle update_text for notes without active codemirror (apply updates in store instead of codemirror) => causes unselected notes to be out of sync
-#   * [ ] on delete: if note is currently selected, navigate away
-# * [ ] awareness
-#   * [ ] frontend: show cursors of other users
-#   * [ ] frontend: show user avatars in notes sidebar
-#   * [ ] frontend: on connect/disconnect: clear local awareness data
-#   * [ ] backend: on connect/disconnect: send event to all clients with user information (id, username, name, random color)
-#   * [ ] frontend: on user connected: send all local awareness information
-#   * [ ] backend: only broadcast awareness information, do not store it
-#   * [ ] awareness info: current page, per-field selections+cursor
-# * [ ] security
-#   * [x] websocket authentication
-#   * [x] permission checks
-#   * [ ] close connection
-#       * [ ] on logout
-#       * [x] on project deletion
-#       * [x] on project set readonly
-#       * [x] on user removed from project
-# * [ ] tests
-#   * [x] test websocket authentication
-#   * [x] test concurrent updates
-#   * [x] test sync to DB
-#   * [x] test sync to DB => history entry
-#   * [x] test API update => update message
-#   * [x] test save signal => update message
-# * [ ] other
-#   * [ ] update NOTICE
-#   * [ ] remove excessive logging
-#   * [ ] remove DemoConsumer
-

deploy/sysreptor.nginx

@@ -24,8 +24,10 @@ server {
 	ssl_stapling on;
 	ssl_stapling_verify on;
 
+  # Large timeouts for long running websocket connections
+	proxy_read_timeout 8h;
+	proxy_send_timeout 8h;
 	client_max_body_size 0;
-	proxy_read_timeout 300;
 
 	location / {
 		include proxy_params;

frontend/src/composables/markdown.ts

@@ -164,7 +164,6 @@ export function useMarkdownEditor({ props, emit, extensions }: {
 
   function onBeforeApplyRemoteTextChange(event: any) {
     if (editorView.value && event.path === props.value.collab?.path) {
-      console.log('Markdown onBeforeRemoteTextChange', event);
       editorView.value.dispatch(editorView.value.state.update({
         changes: event.changes,
         annotations: [
@@ -277,7 +276,6 @@ export function useMarkdownEditor({ props, emit, extensions }: {
 
   watch(valueNotNull, () => {
     if (editorView.value && valueNotNull.value !== editorView.value.state.doc.toString()) {
-      console.log('useMarkdownEditor watch valueNotNull');
       editorView.value.dispatch(editorView.value.state.update({
         changes: {
           from: 0,

frontend/src/utils/collab.ts

@@ -60,7 +60,6 @@ export function useCollab(storeState: CollabStoreState<any>) {
       'ws://localhost:8000' : 
       `${window.location.protocol === 'https' ? 'wss' : 'ws'}://${window.location.host}/`;
     const wsUrl = urlJoin(serverUrl, storeState.websocketPath);
-    console.log('useCollab.connect websocket', wsUrl);
     storeState.perPathState.clear();
     storeState.connectionState = CollabConnectionState.CONNECTING;
     storeState.websocket = new WebSocket(wsUrl);
@@ -75,7 +74,6 @@ export function useCollab(storeState: CollabStoreState<any>) {
     });
     storeState.websocket.addEventListener('message', (event: MessageEvent) => {
       const msgData = JSON.parse(event.data);
-      console.log('Received websocket message:', msgData);
       if (msgData.version && msgData.version > storeState.version) {
         storeState.version = msgData.version;
       }
@@ -103,7 +101,6 @@ export function useCollab(storeState: CollabStoreState<any>) {
     if (storeState.connectionState === CollabConnectionState.CLOSED) {
       return;
     }
-    console.log('useCollab.disconnect websocket');
     storeState.websocket?.close();
     storeState.connectionState = CollabConnectionState.CLOSED;
     storeState.websocket = null;
@@ -111,7 +108,6 @@ export function useCollab(storeState: CollabStoreState<any>) {
   }
 
   function websocketSend(msg: string) {
-    console.log('sendUpdateWebsocket', msg);
     storeState.websocket?.send(msg);
   }