|
@@ -0,0 +1,240 @@
|
|
1
|
+#!/usr/bin/env python
|
|
2
|
+# -*- coding: utf-8 -*-
|
|
3
|
+#
|
|
4
|
+"""
|
|
5
|
+media (Media Tools)
|
|
6
|
+===================
|
|
7
|
+
|
|
8
|
+**Author:**
|
|
9
|
+
|
|
10
|
+* Dirk Alders <sudo-dirk@mount-mockery.de>
|
|
11
|
+
|
|
12
|
+**Description:**
|
|
13
|
+
|
|
14
|
+ This module helps on all issues with media files, like tags (e.g. exif, id3) and transformations.
|
|
15
|
+
|
|
16
|
+**Submodules:**
|
|
17
|
+
|
|
18
|
+* :mod:`mmod.module.sub1`
|
|
19
|
+* :class:`mmod.module.sub2`
|
|
20
|
+* :func:`mmod.module.sub2`
|
|
21
|
+
|
|
22
|
+**Unittest:**
|
|
23
|
+
|
|
24
|
+ See also the :download:`unittest <../../media/_testresults_/unittest.pdf>` documentation.
|
|
25
|
+"""
|
|
26
|
+__DEPENDENCIES__ = []
|
|
27
|
+
|
|
28
|
+import logging
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+logger_name = 'FSTOOLS'
|
|
32
|
+logger = logging.getLogger(logger_name)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+__DESCRIPTION__ = """The Module {\\tt %s} is designed to help on all issues with media files, like tags (e.g. exif, id3) and transformations.
|
|
36
|
+For more Information read the documentation.""" % __name__.replace('_', '\_')
|
|
37
|
+"""The Module Description"""
|
|
38
|
+__INTERPRETER__ = (2, 3)
|
|
39
|
+"""The Tested Interpreter-Versions"""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+def uid(pathname, max_staleness=3600):
|
|
43
|
+ """
|
|
44
|
+ Function returning a unique id for a given file or path.
|
|
45
|
+
|
|
46
|
+ :param str pathname: File or Path name for generation of the uid.
|
|
47
|
+ :param int max_staleness: If a file or path is older than that, we may consider
|
|
48
|
+ it stale and return a different uid - this is a
|
|
49
|
+ dirty trick to work around changes never being
|
|
50
|
+ detected. Default is 3600 seconds, use None to
|
|
51
|
+ disable this trickery. See below for more details.
|
|
52
|
+ :returns: An object that changes value if the file changed,
|
|
53
|
+ None is returned if there were problems accessing the file
|
|
54
|
+ :rtype: str
|
|
55
|
+
|
|
56
|
+ .. note:: Depending on the operating system capabilities and the way the
|
|
57
|
+ file update is done, this function might return the same value
|
|
58
|
+ even if the file has changed. It should be better than just
|
|
59
|
+ using file's mtime though.
|
|
60
|
+ max_staleness tries to avoid the worst for these cases.
|
|
61
|
+
|
|
62
|
+ .. note:: If this function is used for a path, it will stat all pathes and files rekursively.
|
|
63
|
+
|
|
64
|
+ Using just the file's mtime to determine if the file has changed is
|
|
65
|
+ not reliable - if file updates happen faster than the file system's
|
|
66
|
+ mtime granularity, then the modification is not detectable because
|
|
67
|
+ the mtime is still the same.
|
|
68
|
+
|
|
69
|
+ This function tries to improve by using not only the mtime, but also
|
|
70
|
+ other metadata values like file size and inode to improve reliability.
|
|
71
|
+
|
|
72
|
+ For the calculation of this value, we of course only want to use data
|
|
73
|
+ that we can get rather fast, thus we use file metadata, not file data
|
|
74
|
+ (file content).
|
|
75
|
+
|
|
76
|
+ >>> print 'UID:', uid(__file__)
|
|
77
|
+ UID: 16a65cc78e1344e596ef1c9536dab2193a402934
|
|
78
|
+ """
|
|
79
|
+ if os.path.isdir(pathname):
|
|
80
|
+ pathlist = dirlist(pathname) + filelist(pathname)
|
|
81
|
+ pathlist.sort()
|
|
82
|
+ else:
|
|
83
|
+ pathlist = [pathname]
|
|
84
|
+ uid = []
|
|
85
|
+ for element in pathlist:
|
|
86
|
+ try:
|
|
87
|
+ st = os.stat(element)
|
|
88
|
+ except (IOError, OSError):
|
|
89
|
+ uid.append(None) # for permanent errors on stat() this does not change, but
|
|
90
|
+ # having a changing value would be pointless because if we
|
|
91
|
+ # can't even stat the file, it is unlikely we can read it.
|
|
92
|
+ else:
|
|
93
|
+ fake_mtime = int(st.st_mtime)
|
|
94
|
+ if not st.st_ino and max_staleness:
|
|
95
|
+ # st_ino being 0 likely means that we run on a platform not
|
|
96
|
+ # supporting it (e.g. win32) - thus we likely need this dirty
|
|
97
|
+ # trick
|
|
98
|
+ now = int(time.time())
|
|
99
|
+ if now >= st.st_mtime + max_staleness:
|
|
100
|
+ # keep same fake_mtime for each max_staleness interval
|
|
101
|
+ fake_mtime = int(now / max_staleness) * max_staleness
|
|
102
|
+ uid.append((
|
|
103
|
+ st.st_mtime, # might have a rather rough granularity, e.g. 2s
|
|
104
|
+ # on FAT, 1s on ext3 and might not change on fast
|
|
105
|
+ # updates
|
|
106
|
+ st.st_ino, # inode number (will change if the update is done
|
|
107
|
+ # by e.g. renaming a temp file to the real file).
|
|
108
|
+ # not supported on win32 (0 ever)
|
|
109
|
+ st.st_size, # likely to change on many updates, but not
|
|
110
|
+ # sufficient alone
|
|
111
|
+ fake_mtime) # trick to workaround file system / platform
|
|
112
|
+ # limitations causing permanent trouble
|
|
113
|
+ )
|
|
114
|
+ if sys.version_info < (3, 0):
|
|
115
|
+ secret = ''
|
|
116
|
+ return hmac.new(secret, repr(uid), hashlib.sha1).hexdigest()
|
|
117
|
+ else:
|
|
118
|
+ secret = b''
|
|
119
|
+ return hmac.new(secret, bytes(repr(uid), 'latin-1'), hashlib.sha1).hexdigest()
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+def uid_filelist(path='.', expression='*', rekursive=True):
|
|
123
|
+ SHAhash = hashlib.md5()
|
|
124
|
+ #
|
|
125
|
+ fl = filelist(path, expression, rekursive)
|
|
126
|
+ fl.sort()
|
|
127
|
+ for f in fl:
|
|
128
|
+ if sys.version_info < (3, 0):
|
|
129
|
+ with open(f, 'rb') as fh:
|
|
130
|
+ SHAhash.update(hashlib.md5(fh.read()).hexdigest())
|
|
131
|
+ else:
|
|
132
|
+ with open(f, mode='rb') as fh:
|
|
133
|
+ d = hashlib.md5()
|
|
134
|
+ for buf in iter(partial(fh.read, 128), b''):
|
|
135
|
+ d.update(buf)
|
|
136
|
+ SHAhash.update(d.hexdigest().encode())
|
|
137
|
+ #
|
|
138
|
+ return SHAhash.hexdigest()
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+def filelist(path='.', expression='*', rekursive=True):
|
|
142
|
+ """
|
|
143
|
+ Function returning a list of files below a given path.
|
|
144
|
+
|
|
145
|
+ :param str path: folder which is the basepath for searching files.
|
|
146
|
+ :param str expression: expression to fit including shell-style wildcards.
|
|
147
|
+ :param bool rekursive: search all subfolders if True.
|
|
148
|
+ :returns: list of filenames including the pathe
|
|
149
|
+ :rtype: list
|
|
150
|
+
|
|
151
|
+ .. note:: The returned filenames could be relative pathes depending on argument path.
|
|
152
|
+
|
|
153
|
+ >>> for filename in filelist(path='.', expression='*.py*', rekursive=True):
|
|
154
|
+ ... print filename
|
|
155
|
+ ./__init__.py
|
|
156
|
+ ./__init__.pyc
|
|
157
|
+ """
|
|
158
|
+ li = list()
|
|
159
|
+ if os.path.exists(path):
|
|
160
|
+ logger.debug('FILELIST: path (%s) exists - looking for files to append', path)
|
|
161
|
+ for filename in glob.glob(os.path.join(path, expression)):
|
|
162
|
+ if os.path.isfile(filename):
|
|
163
|
+ li.append(filename)
|
|
164
|
+ for directory in os.listdir(path):
|
|
165
|
+ directory = os.path.join(path, directory)
|
|
166
|
+ if os.path.isdir(directory) and rekursive and not os.path.islink(directory):
|
|
167
|
+ li.extend(filelist(directory, expression))
|
|
168
|
+ else:
|
|
169
|
+ logger.warning('FILELIST: path (%s) does not exist - empty filelist will be returned', path)
|
|
170
|
+ return li
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+def dirlist(path='.', rekursive=True):
|
|
174
|
+ """
|
|
175
|
+ Function returning a list of directories below a given path.
|
|
176
|
+
|
|
177
|
+ :param str path: folder which is the basepath for searching files.
|
|
178
|
+ :param bool rekursive: search all subfolders if True.
|
|
179
|
+ :returns: list of filenames including the pathe
|
|
180
|
+ :rtype: list
|
|
181
|
+
|
|
182
|
+ .. note:: The returned filenames could be relative pathes depending on argument path.
|
|
183
|
+
|
|
184
|
+ >>> for dirname in dirlist(path='..', rekursive=True):
|
|
185
|
+ ... print dirname
|
|
186
|
+ ../caching
|
|
187
|
+ ../fstools
|
|
188
|
+ """
|
|
189
|
+ li = list()
|
|
190
|
+ if os.path.exists(path):
|
|
191
|
+ logger.debug('DIRLIST: path (%s) exists - looking for directories to append', path)
|
|
192
|
+ for dirname in os.listdir(path):
|
|
193
|
+ fulldir = os.path.join(path, dirname)
|
|
194
|
+ if os.path.isdir(fulldir):
|
|
195
|
+ li.append(fulldir)
|
|
196
|
+ if rekursive:
|
|
197
|
+ li.extend(dirlist(fulldir))
|
|
198
|
+ else:
|
|
199
|
+ logger.warning('DIRLIST: path (%s) does not exist - empty filelist will be returned', path)
|
|
200
|
+ return li
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+def is_writeable(path):
|
|
204
|
+ """.. warning:: Needs to be documented
|
|
205
|
+ """
|
|
206
|
+ if os.access(path, os.W_OK):
|
|
207
|
+ # path is writable whatever it is, file or directory
|
|
208
|
+ return True
|
|
209
|
+ else:
|
|
210
|
+ # path is not writable whatever it is, file or directory
|
|
211
|
+ return False
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+def mkdir(path):
|
|
215
|
+ """.. warning:: Needs to be documented
|
|
216
|
+ """
|
|
217
|
+ path = os.path.abspath(path)
|
|
218
|
+ if not os.path.exists(os.path.dirname(path)):
|
|
219
|
+ mkdir(os.path.dirname(path))
|
|
220
|
+ if not os.path.exists(path):
|
|
221
|
+ os.mkdir(path)
|
|
222
|
+ return os.path.isdir(path)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+def open_locked_non_blocking(*args, **kwargs):
|
|
226
|
+ """.. warning:: Needs to be documented (acquire exclusive lock file access). Throws an exception, if file is locked!
|
|
227
|
+ """
|
|
228
|
+ import fcntl
|
|
229
|
+ locked_file_descriptor = open(*args, **kwargs)
|
|
230
|
+ fcntl.lockf(locked_file_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
231
|
+ return locked_file_descriptor
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+def open_locked_blocking(*args, **kwargs):
|
|
235
|
+ """.. warning:: Needs to be documented (acquire exclusive lock file access). Blocks until file is free. deadlock!
|
|
236
|
+ """
|
|
237
|
+ import fcntl
|
|
238
|
+ locked_file_descriptor = open(*args, **kwargs)
|
|
239
|
+ fcntl.lockf(locked_file_descriptor, fcntl.LOCK_EX)
|
|
240
|
+ return locked_file_descriptor
|