diff --git a/.gitignore b/.gitignore index 5d381cc..c7e9395 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +asyncron/static/highlight + + # ---> Python # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/asyncron/base/admin.py b/asyncron/base/admin.py index 0385e8f..4a0b42b 100644 --- a/asyncron/base/admin.py +++ b/asyncron/base/admin.py @@ -10,7 +10,7 @@ class BaseModelAdmin( admin.ModelAdmin ): def formfield_for_dbfield( self, db_field, **kwargs ): formfield = super().formfield_for_dbfield(db_field, **kwargs) - if isinstance( formfield, JSONField ): + if isinstance( formfield, JSONField ) and not isinstance( formfield.widget, Codearea ): formfield.widget = Codearea( language = "json-pretty" ) custom_widget = getattr(self, "field_widgets", {}).get(db_field.name, None) diff --git a/asyncron/base/models.py b/asyncron/base/models.py index 0ce879b..e7820d3 100644 --- a/asyncron/base/models.py +++ b/asyncron/base/models.py @@ -45,7 +45,7 @@ class BaseModel( models.Model ): except: print("WARNING: could not check already cached relations.") if fields: - await sync_to_async(lambda: [ getattr(self, f) for f in fields ])() + await sync_to_async(lambda: [ rgetattr(self, f) for f in fields ])() @classmethod diff --git a/asyncron/fields/__init__.py b/asyncron/fields/__init__.py new file mode 100644 index 0000000..8659d03 --- /dev/null +++ b/asyncron/fields/__init__.py @@ -0,0 +1,3 @@ + +from .debug import DebugJSONField +from .toml import TOMLField diff --git a/asyncron/fields/debug.py b/asyncron/fields/debug.py new file mode 100644 index 0000000..41b654d --- /dev/null +++ b/asyncron/fields/debug.py @@ -0,0 +1,54 @@ + +from django.db import models + +class DebugJSONField( models.JSONField ): + + def check(self, *args, **kwargs): + result = super().check(*args, **kwargs) + print("Check:", args, kwargs, result) + return result + + def deconstruct(self, *args, **kwargs): + result = super().deconstruct(*args, **kwargs) + print("deconstruct:", args, kwargs, result) + return result + + def from_db_value(self, *args, **kwargs): + result = super().from_db_value(*args, **kwargs) + print("from_db_value:", args, kwargs, result) + return result + + def get_internal_type(self, *args, **kwargs): + result = super().get_internal_type(*args, **kwargs) + print("get_internal_type:", args, kwargs, result) + return result + + def get_db_prep_value(self, *args, **kwargs): + result = super().get_db_prep_value(*args, **kwargs) + print("get_db_prep_value:", args, kwargs, result) + return result + + def get_db_prep_save(self, *args, **kwargs): + result = super().get_db_prep_save(*args, **kwargs) + print("get_db_prep_save:", args, kwargs, result) + return result + + def get_transform(self, *args, **kwargs): + result = super().get_transform(*args, **kwargs) + print("get_transform:", args, kwargs, result) + return result + + def validate(self, *args, **kwargs): + result = super().validate(*args, **kwargs) + print("validate:", args, kwargs, result) + return result + + def value_to_string(self, *args, **kwargs): + result = super().value_to_string(*args, **kwargs) + print("value_to_string:", args, kwargs, result) + return result + + def formfield(self, *args, **kwargs): + result = super().formfield(*args, **kwargs) + print("formfield:", args, kwargs, result) + return result diff --git a/asyncron/fields/toml.py b/asyncron/fields/toml.py new file mode 100644 index 0000000..305901d --- /dev/null +++ b/asyncron/fields/toml.py @@ -0,0 +1,83 @@ +from django.db import models +from django.forms import fields + +from asyncron.widgets import Codearea + +import tomlkit +from tomlkit.exceptions import ParseError +from tomlkit.toml_document import TOMLDocument + + +class InvalidTOMLInput(str): + pass + +class TOMLFormField( fields.JSONField ): + default_error_messages = { + "invalid": "Enter a valid TOML.", + } + + widget = Codearea( language = "ini" ) + + def to_python(self, value): + if self.disabled: return value + if isinstance(value, (list, dict, int, float)): return value + + try: + converted = tomlkit.loads( value ) + except ParseError: + raise ValidationError( + self.error_messages["invalid"], + code="invalid", + params={"value": value}, + ) + return converted + + + def bound_data(self, data, initial): + if self.disabled: return initial + if data is None: return None + + try: + return tomlkit.loads( data ) + except ParseError: + return InvalidTOMLInput(data) + + def prepare_value(self, value): + if isinstance(value, InvalidTOMLInput): return value + return value.as_string() + + + +class TOMLField( models.JSONField ): + empty_strings_allowed = True + description = "A TOML Field" + + def from_db_value(self, *args, **kwargs): + value_as_dict = super().from_db_value(*args, **kwargs) + + #This is a cop-out, but I don't have time to do this properly even though it's very interesting. + #But duplicating the date on fields that are meant to be human readable, isn't a serious problem. + # + # Links to related parts of tomlkit in case I decide to improve on this later: + # - as_string function which decies which renderer to choose + # - https://github.com/python-poetry/tomlkit/blob/6042e0ce80c8c49f325a6e60b6ee3e153669b144/tomlkit/container.py#L485 + # + # - The simple renderer which has the item.as_string bit that we have to optimize out. + # - https://github.com/python-poetry/tomlkit/blob/6042e0ce80c8c49f325a6e60b6ee3e153669b144/tomlkit/container.py#L633 + # + try: value_as_toml = tomlkit.loads( value_as_dict.pop('_toml', '') ) + except: value_as_toml = tomlkit.document() + value_as_toml.update( value_as_dict ) + + return value_as_toml + + def get_prep_value(self, *args, **kwargs): + result = super().get_prep_value( *args, **kwargs ) + if not isinstance( result, TOMLDocument ): return result + as_dict = result.unwrap() + as_dict['_toml'] = result.as_string() + return as_dict + + def formfield( self, *args, **kwargs ): + kwargs['form_class'] = TOMLFormField + return super().formfield(*args, **kwargs) diff --git a/asyncron/static/extract-highlight-js-here b/asyncron/static/extract-highlight-js-here new file mode 100644 index 0000000..e69de29 diff --git a/asyncron/widgets.py b/asyncron/widgets.py index eeaeed0..1688a1a 100644 --- a/asyncron/widgets.py +++ b/asyncron/widgets.py @@ -1,5 +1,5 @@ from django.template import loader -from django.forms import Textarea +from django.forms.widgets import Textarea import json diff --git a/asyncron/workers.py b/asyncron/workers.py index 68ac6f3..e8d0677 100644 --- a/asyncron/workers.py +++ b/asyncron/workers.py @@ -86,6 +86,7 @@ class AsyncronWorker: self.clearing_dead_workers = False self.watching_models = collections.defaultdict( set ) # Model -> Set of key name of the tasks self.work_loop_over = asyncio.Event() + self.database_unreachable = False if daemon: self.thread = threading.Thread( target = self.start ) @@ -205,9 +206,14 @@ class AsyncronWorker: last_overtake_attempt = 0 current_master = False + while True: try: await Worker.objects.filter( is_master = False ).aupdate( is_master = models.Q(id = self.model.id) ) + except RuntimeError as e: #Syntax Error cause: cannot schedule new futures after interpreter shutdown + if "interpreter shutdown" not in e.args[0]: raise + self.database_unreachable = True + break except IntegrityError: # I'm not master! loop_wait = 5 + random.random() * 15 @@ -242,8 +248,9 @@ class AsyncronWorker: await self.clear_orphaned_traces() finally: - await Worker.objects.filter( id = self.model.id ).aupdate( last_crowning_attempt = timezone.now() ) - await asyncio.sleep( loop_wait ) + if not self.database_unreachable: + await Worker.objects.filter( id = self.model.id ).aupdate( last_crowning_attempt = timezone.now() ) + await asyncio.sleep( loop_wait ) async def clear_orphaned_traces( self ): from .models import Worker, Task, Trace @@ -277,7 +284,13 @@ class AsyncronWorker: await func.task.arefresh_from_db() else: #For now, to commit changes to db init_task.id = func.task.id + + #DEBUG this: + #django.db.utils.IntegrityError: insert or update on table "asyncron_task" violates foreign key constraint "asyncron_task_worker_lock_id_0bb55026_fk_asyncron_worker_id" + #DETAIL: Key (worker_lock_id)=(6035) is not present in table "asyncron_worker". await init_task.asave() + #END of DEBUG! + await func.task.arefresh_from_db() @@ -286,7 +299,9 @@ class AsyncronWorker: self.check_interval = 0 - while await Worker.objects.filter( id = self.model.id ).aexists(): + while not self.database_unreachable: + + if not await Worker.objects.filter( id = self.model.id ).aexists(): break await asyncio.sleep( self.check_interval ) self.check_interval = 10 diff --git a/requirements.dev.txt b/requirements.dev.txt index 0df1984..642e659 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,2 +1,3 @@ django humanize +tomlkit