From 3061da63ade71420769d33094123fec9b32b757b Mon Sep 17 00:00:00 2001 From: Dirk Alders Date: Fri, 31 Jan 2020 00:22:27 +0100 Subject: [PATCH] Usage of pylib media for getting metadata from media files --- admin.py | 6 +- help.py | 3 +- migrations/0003_auto_20200130_2203.py | 22 +++ models.py | 260 +++++--------------------- search.py | 3 +- views/infoviews.py | 2 +- 6 files changed, 73 insertions(+), 223 deletions(-) create mode 100644 migrations/0003_auto_20200130_2203.py diff --git a/admin.py b/admin.py index 6862234..91d3bcd 100644 --- a/admin.py +++ b/admin.py @@ -22,8 +22,7 @@ class ItemAdmin(admin.ModelAdmin): 'num_other_c', 'num_videos_c', 'sil_c', - 'camera_vendor_c', - 'camera_model_c', + 'camera_c', 'width_c', 'height_c', 'exposure_program_c', @@ -75,8 +74,7 @@ class ItemAdmin(admin.ModelAdmin): ] if obj.type == TYPE_IMAGE: rv += [ - 'camera_vendor_c', - 'camera_model_c', + 'camera_c', 'width_c', 'height_c', 'lon_c', diff --git a/help.py b/help.py index c1909e7..eeae1c0 100644 --- a/help.py +++ b/help.py @@ -74,8 +74,7 @@ more in the different type depending lists. * lat (NUMERIC): * height (NUMERIC): * iso (NUMERIC): -* camera_vendor (TEXT): -* camera_model (TEXT): +* camera (TEXT): * orientation (NUMERIC): * width (NUMERIC): diff --git a/migrations/0003_auto_20200130_2203.py b/migrations/0003_auto_20200130_2203.py new file mode 100644 index 0000000..66dc125 --- /dev/null +++ b/migrations/0003_auto_20200130_2203.py @@ -0,0 +1,22 @@ +# Generated by Django 2.2.9 on 2020-01-30 22:03 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('pygal', '0002_setting_suspend_puplic'), + ] + + operations = [ + migrations.RenameField( + model_name='item', + old_name='camera_model_c', + new_name='camera_c', + ), + migrations.RemoveField( + model_name='item', + name='camera_vendor_c', + ), + ] diff --git a/models.py b/models.py index be5c181..c4ea1d5 100644 --- a/models.py +++ b/models.py @@ -6,8 +6,8 @@ from django.urls import reverse import fstools import json import logging +import media import os -from PIL import Image import pygal import subprocess import time @@ -76,33 +76,6 @@ def get_item_by_rel_path(rel_path): return i -def ffprobe_lines(full_path): - def _ffprobe_command(full_path): - return ['ffprobe', '-v', 'quiet', '-show_format', '-show_streams', full_path] - - def _avprobe_command(full_path): - return ['avprobe', '-v', 'quiet', '-show_format', '-show_streams', full_path] - - def decode(string): - for i in ['utf-8', 'cp1252']: - try: - return string.decode(i) - except UnicodeEncodeError: - pass - except UnicodeDecodeError: - pass - return string - - try: - try: - text_to_be_parsed = subprocess.check_output(_avprobe_command(full_path)) - except OSError: - text_to_be_parsed = subprocess.check_output(_ffprobe_command(full_path)) - except subprocess.CalledProcessError: - text_to_be_parsed = '' - return decode(text_to_be_parsed).splitlines() - - def is_valid_area(x1, y1, x2, y2): for p in [x1, y1, x2, y2]: if type(p) is not int: @@ -125,8 +98,7 @@ class ItemData(dict): 'lat', 'height', 'iso', - 'camera_vendor', - 'camera_model', + 'camera', 'orientation', 'width', 'duration', @@ -154,8 +126,7 @@ class ItemData(dict): 'f_number': 0, 'focal_length': 0, 'iso': 0, - 'camera_vendor': '-', - 'camera_model': '-', + 'camera': '-', 'orientation': 1, 'duration': 3, 'ratio': 1, @@ -224,8 +195,7 @@ class Item(models.Model): lat_c = models.FloatField(null=True, blank=True) height_c = models.IntegerField(null=True, blank=True) iso_c = models.IntegerField(null=True, blank=True) - camera_vendor_c = models.CharField(max_length=100, null=True, blank=True) - camera_model_c = models.CharField(max_length=100, null=True, blank=True) + camera_c = models.CharField(max_length=100, null=True, blank=True) orientation_c = models.IntegerField(null=True, blank=True) width_c = models.IntegerField(null=True, blank=True) # video @@ -247,6 +217,31 @@ class Item(models.Model): num_videos_c = models.IntegerField(null=True, blank=True) sil_c = models.TextField(null=True, blank=True) + MODEL_TO_MEDIA_DATA = { + media.metadata.KEY_SIZE: 'size_c', + media.metadata.KEY_TIME: 'datetime_c', + media.metadata.KEY_EXPOSURE_PROGRAM: 'exposure_program_c', + media.metadata.KEY_EXPOSURE_TIME: 'exposure_time_c', + media.metadata.KEY_FLASH: 'flash_c', + media.metadata.KEY_APERTURE: 'f_number_c', + media.metadata.KEY_FOCAL_LENGTH: 'focal_length_c', + media.metadata.KEY_GPS: {'lon': 'lon_c', 'lat': 'lat_c'}, + media.metadata.KEY_HEIGHT: 'height_c', + media.metadata.KEY_ISO: 'iso_c', + media.metadata.KEY_CAMERA: 'camera_c', + media.metadata.KEY_ORIENTATION: 'orientation_c', + media.metadata.KEY_WIDTH: 'width_c', + media.metadata.KEY_DURATION: 'duration_c', + media.metadata.KEY_RATIO: 'ratio_c', + media.metadata.KEY_ALBUM: 'album_c', + media.metadata.KEY_ARTIST: 'artist_c', + media.metadata.KEY_BITRATE: 'bitrate_c', + media.metadata.KEY_GENRE: 'genre_c', + media.metadata.KEY_TITLE: 'title_c', + media.metadata.KEY_TRACK: 'track_c', + media.metadata.KEY_YEAR: 'year_c', + } + def __init__(self, *args, **kwargs): self.__current_uid__ = None self.__current_settings__ = None @@ -378,53 +373,15 @@ class Item(models.Model): self.uid_c = self.current_uid() self.settings_c = self.current_settings() self.data_version_c = self.DATA_VERSION_NUMBER - if self.type == TYPE_AUDIO: - self.__update_audio_file_data__(full_path) - elif self.type == TYPE_FOLDER: + if self.type == TYPE_FOLDER: self.__update_folder_file_data__(full_path) - elif self.type == TYPE_IMAGE: - self.__update_image_file_data__(full_path) elif self.type == TYPE_OTHER: self.__update_other_file_data__(full_path) - elif self.type == TYPE_VIDEO: - self.__update_video_file_data__(full_path) + else: + self.__update_media_file_data__(full_path) for key, value in self.cached_item_data.items(): logger.debug(' - Adding %s=%s', key, repr(value)) - def __update_audio_file_data__(self, full_path): - self.size_c = os.path.getsize(full_path) - # - tag_type_target_dict = {} - tag_type_target_dict['album'] = (str, 'album') - tag_type_target_dict['TAG:album'] = (str, 'album') - tag_type_target_dict['TAG:artist'] = (str, 'artist') - tag_type_target_dict['artist'] = (str, 'artist') - tag_type_target_dict['bit_rate'] = (self.__int_conv__, 'bitrate') - tag_type_target_dict['duration'] = (float, 'duration') - tag_type_target_dict['TAG:genre'] = (str, 'genre') - tag_type_target_dict['genre'] = (str, 'genre') - tag_type_target_dict['TAG:title'] = (str, 'title') - tag_type_target_dict['title'] = (str, 'title') - tag_type_target_dict['TAG:track'] = (self.__int_conv__, 'track') - tag_type_target_dict['track'] = (self.__int_conv__, 'track') - tag_type_target_dict['TAG:date'] = (self.__int_conv__, 'year') - tag_type_target_dict['date'] = (self.__int_conv__, 'year') - for line in ffprobe_lines(full_path): - try: - key, val = [snippet.strip() for snippet in line.split('=')] - except ValueError: - continue - else: - if key in tag_type_target_dict: - tp, name = tag_type_target_dict[key] - try: - setattr(self, name + '_c', tp(val)) - except ValueError: - logger.log(logging.WARNING if val else logger.INFO, 'Can\'t convert %s (%s) for %s', repr(val), name, self.name) - # - if self.year_c is not None and self.track_c is not None: - self.datetime_c = datetime.datetime(max(1971, self.year_c), 1, 1, 12, 0, (60 - self.track_c) % 60, tzinfo=timezone.utc) - def __update_folder_file_data__(self, full_path): sil = [] self.size_c = 0 @@ -462,151 +419,26 @@ class Item(models.Model): if len(sil) > 0: self.datetime_c = sil[0].datetime_c - def __update_image_file_data__(self, full_path): - self.size_c = os.path.getsize(full_path) - # - tag_type_target_dict = {} - tag_type_target_dict[0x9003] = (self.__datetime_conv__, 'datetime') - tag_type_target_dict[0x8822] = (self.__exposure_program_conv__, 'exposure_program') - tag_type_target_dict[0x829A] = (self.__num_denum_conv__, 'exposure_time') - tag_type_target_dict[0x9209] = (self.__flash_conv__, 'flash') - tag_type_target_dict[0x829D] = (self.__num_denum_conv__, 'f_number') - tag_type_target_dict[0x920A] = (self.__num_denum_conv__, 'focal_length') - tag_type_target_dict[0x8825] = (self.__gps_conv__, ('lon', 'lat')) - tag_type_target_dict[0xA003] = (self.__int_conv__, 'height') - tag_type_target_dict[0x8827] = (self.__int_conv__, 'iso') - tag_type_target_dict[0x010F] = (str, 'camera_vendor') - tag_type_target_dict[0x0110] = (str, 'camera_model') - tag_type_target_dict[0x0112] = (self.__int_conv__, 'orientation') - tag_type_target_dict[0xA002] = (self.__int_conv__, 'width') - im = Image.open(full_path) - try: - exif = dict(im._getexif().items()) - except AttributeError: - logger.debug('%s does not have any exif information', full_path) - else: - for key in tag_type_target_dict: - if key in exif: - tp, name = tag_type_target_dict[key] - if type(name) is tuple: - data = tp(exif[key]) or (None, None) - for name, val in zip(name, data): - setattr(self, name + '_c', val) - else: - setattr(self, name + '_c', tp(exif[key])) + def __update_media_file_data__(self, full_path): + data = media.get_media_data(full_path) or {} + for key in self.MODEL_TO_MEDIA_DATA: + value = data.get(key) + if key == media.metadata.KEY_GPS: # Split GPS data in lon and lat + if value is not None: + for k in self.MODEL_TO_MEDIA_DATA[key]: + setattr(self, self.MODEL_TO_MEDIA_DATA[key][k], value[k]) + else: + if key == media.metadata.KEY_TIME: # convert time to datetime + if data.get(media.metadata.KEY_TIME_IS_SUBSTITUTION) and self.type == TYPE_IMAGE: # don't use time substitution for images + break + value = datetime.datetime.fromtimestamp(value, tz=datetime.timezone.utc) + setattr(self, self.MODEL_TO_MEDIA_DATA[key], value) def __update_other_file_data__(self, full_path): self.size_c = os.path.getsize(full_path) # self.datetime_c = datetime.datetime.fromtimestamp(os.path.getctime(full_path), tz=timezone.utc) - def __update_video_file_data__(self, full_path): - self.size_c = os.path.getsize(full_path) - # - tag_type_target_dict = {} - tag_type_target_dict['creation_time'] = (self.__vid_datetime_conv__, 'datetime') - tag_type_target_dict['TAG:creation_time'] = (self.__vid_datetime_conv__, 'datetime') - tag_type_target_dict['duration'] = (float, 'duration') - tag_type_target_dict['height'] = (self.__int_conv__, 'height') - tag_type_target_dict['width'] = (self.__int_conv__, 'width') - tag_type_target_dict['display_aspect_ratio'] = (self.__ratio_conv__, 'ratio') - for line in ffprobe_lines(full_path): - try: - key, val = [snippet.strip() for snippet in line.split('=')] - except ValueError: - continue - else: - if key in tag_type_target_dict: - tp, name = tag_type_target_dict[key] - try: - setattr(self, name + '_c', tp(val)) - except ValueError: - logger.log(logging.WARNING if val else logger.INFO, 'Can\'t convert %s (%s) for %s', repr(val), name, self.name) - - def __datetime_conv__(self, dt): - format_string = "%Y:%m:%d %H:%M:%S%z" - return datetime.datetime.strptime(dt + '+0000', format_string) - - def __exposure_program_conv__(self, n): - return { - 0: 'Unidentified', - 1: 'Manual', - 2: 'Program Normal', - 3: 'Aperture Priority', - 4: 'Shutter Priority', - 5: 'Program Creative', - 6: 'Program Action', - 7: 'Portrait Mode', - 8: 'Landscape Mode' - }.get(n, '-') - - def __flash_conv__(self, n): - return { - 0: 'No', - 1: 'Fired', - 5: 'Fired (?)', # no return sensed - 7: 'Fired (!)', # return sensed - 9: 'Fill Fired', - 13: 'Fill Fired (?)', - 15: 'Fill Fired (!)', - 16: 'Off', - 24: 'Auto Off', - 25: 'Auto Fired', - 29: 'Auto Fired (?)', - 31: 'Auto Fired (!)', - 32: 'Not Available' - }.get(n, '-') - - def __int_conv__(self, value): - try: - return int(value) - except ValueError: - for c in ['.', '/', '-']: - p = value.find(c) - if p >= 0: - value = value[:p] - if value == '': - return 0 - return int(value) - - def __num_denum_conv__(self, data): - num, denum = data - return num / denum - - def __gps_conv__(self, data): - def lat_lon_cal(lon_or_lat): - lon_lat = 0. - fac = 1. - for num, denum in lon_or_lat: - lon_lat += float(num) / float(denum) * fac - fac *= 1. / 60. - return lon_lat - try: - lon = lat_lon_cal(data[0x0004]) - lat = lat_lon_cal(data[0x0002]) - if lon != 0 or lat != 0: # do not use lon and lat equal 0, caused by motorola gps weakness - return lon, lat - except KeyError: - logger.warning('GPS data extraction failed for %s: %s', self.name, repr(data)) - return None - - def __vid_datetime_conv__(self, dt): - try: - dt = dt[:dt.index('.')] - except ValueError: - pass # time string seems to have no '.' - dt = dt.replace('T', ' ').replace('/', '').replace('\\', '') - if len(dt) == 16: - dt += ':00' - dt += '+0000' - format_string = '%Y-%m-%d %H:%M:%S%z' - return datetime.datetime.strptime(dt, format_string) - - def __ratio_conv__(self, ratio): - ratio = ratio.replace('\\', '') - num, denum = ratio.split(':') - return float(num) / float(denum) - def __str__(self): return 'Item: %s' % self.rel_path diff --git a/search.py b/search.py index 533deda..69a5618 100644 --- a/search.py +++ b/search.py @@ -31,8 +31,7 @@ SCHEMA = Schema( lat=NUMERIC, height=NUMERIC, iso=NUMERIC, - camera_vendor=TEXT, - camera_model=TEXT, + camera=TEXT, orientation=NUMERIC, width=NUMERIC, # Audio Cache diff --git a/views/infoviews.py b/views/infoviews.py index 88b6cd9..323f6f3 100644 --- a/views/infoviews.py +++ b/views/infoviews.py @@ -75,7 +75,7 @@ class image_view(base_view): @property def item_information(self): rv = [] - rv.append({'description': _('Camera'), 'data': '%s - %s' % (self.item.item_data.camera_vendor, self.item.item_data.camera_model), 'url': None}) + rv.append({'description': _('Camera'), 'data': '%s' % (self.item.item_data.camera), 'url': None}) rv.append({'description': _('Resolution'), 'data': '%d x %d' % (self.item.item_data.width, self.item.item_data.height), 'url': None}) gps_data = self.item.item_data.gps if gps_data is not None: