From 92d532b940a25aff09ca97c53d969ffa1e172efa Mon Sep 17 00:00:00 2001 From: Andy King Date: Sun, 27 Jul 2014 12:33:37 -0400 Subject: [PATCH] Added util folder to hold the mimetypes variable generator and default mime-types as harvested from libmagic. Added dictionary of mime-types -> modules that might have identified it --- magic.py | 253 ++++++++++++++++++ util/fileMimes | 619 +++++++++++++++++++++++++++++++++++++++++++ util/parse_magdir.py | 52 ++++ 3 files changed, 924 insertions(+) create mode 100644 util/fileMimes create mode 100644 util/parse_magdir.py diff --git a/magic.py b/magic.py index 10685ac..34af55c 100644 --- a/magic.py +++ b/magic.py @@ -58,6 +58,259 @@ def __init__(self, mime=False, magic_file=None, mime_encoding=False, self.thread = threading.currentThread() + #Default mime-types for libmagic in an easy to access dictionary + self.file_types = {'audio/x-dec-basic': ['audio'], + 'audio/mp4': ['animation'], + 'application/vnd.oasis.opendocument.chart-template': ['archive'], + 'image/x-polar-monitor-bitmap': ['images'], + 'application/x-ichitaro6': ['wordprocessors'], + 'video/quicktime': ['animation'], + 'application/vnd.oasis.opendocument.graphics': ['archive'], + 'application/dicom': ['images'], + 'application/x-gnumeric': ['gnumeric'], + 'application/x-java-applet': ['cafebabe'], + 'text/texmacs': ['lisp'], + 'application/mac-binhex40': ['macintosh'], + 'application/x-msaccess': ['database'], + 'text/x-texinfo': ['tex'], + 'application/x-java-pack200': ['cafebabe'], + 'application/x-dbf': ['database'], + 'application/x-123': ['msdos'], + 'text/x-perl': ['perl'], + 'text/x-python': ['python'], + 'application/x-epoc-opl': ['epoc'], + 'image/x-portable-greymap': ['images'], + 'image/gif': ['images'], + 'application/xml-sitemap': ['sgml'], + 'application/x-stuffit': ['macintosh'], + 'application/x-kdelnk': ['kde'], + 'text/x-makefile': ['make'], + 'text/x-asm': ['assembler'], + 'application/x-scribus': ['wordprocessors'], + 'application/x-font-ttf': ['fonts'], + 'video/3gpp': ['animation'], + 'image/x-coreldraw': ['riff'], + 'audio/x-aiff': ['iff'], + 'image/x-award-bmp': ['images'], + 'video/webm': ['matroska'], + 'image/x-paintnet': ['images'], + 'image/x-unknown': ['images'], + 'image/jpx': ['jpeg'], + 'video/mp4': ['animation'], + 'application/vnd.oasis.opendocument.image': ['archive'], + 'text/x-c++': ['c-lang'], + 'application/x-ichitaro4': ['wordprocessors'], + 'application/x-lha': ['archive', 'msdos'], + 'text/x-lua': ['tcl', 'lua'], + 'application/pgp-signature': ['pgp'], + 'application/xml': ['sgml'], + 'text/x-vcard': ['misctools'], + 'text/x-fortran': ['fortran'], + 'application/x-bittorrent': ['archive'], + 'model/vrml': ['animation'], + 'application/x-cpio': ['archive'], + 'audio/x-musepack': ['audio'], + 'application/x-quicktime-player': ['animation'], + 'application/x-freemind': ['wordprocessors'], + 'application/x-epoc-data': ['epoc'], + 'application/vnd.oasis.opendocument.text-web': ['archive'], + 'audio/x-unknown': ['audio'], + 'application/x-tokyocabinet-btree': ['database'], + 'audio/x-adpcm': ['audio'], + 'audio/midi': ['audio'], + 'application/pdf': ['pdf'], + 'application/vnd.oasis.opendocument.text': ['archive'], + 'application/pgp': ['pgp'], + 'image/x-xcursor': ['xwindows'], + 'image/x-xcf': ['gimp'], + 'application/vnd.oasis.opendocument.database': ['archive'], + 'image/jp2': ['animation', 'jpeg'], + 'application/x-font-sfn': ['fonts'], + 'image/svg+xml': ['sgml'], + 'application/jar': ['archive'], + 'image/x-canon-cr2': ['images'], + 'text/PGP': ['pgp', 'gnu'], + 'application/x-bzip2': ['compress'], + 'application/x-iso9660-image': ['filesystems'], + 'application/x-epoc-jotter': ['epoc'], + 'image/x-icon': ['msdos'], + 'audio/basic': ['audio'], + 'chemical/x-pdb': ['scientific'], + 'application/x-dvi': ['tex'], + 'image/x-portable-bitmap': ['images'], + 'application/x-mif': ['frame'], + 'video/x-sgi-movie': ['animation'], + 'audio/x-pn-realaudio': ['audio'], + 'application/postscript': ['printer'], + 'application/vnd.oasis.opendocument.text-master': ['archive'], + 'model/x3d': ['animation'], + 'video/x-ms-asf': ['animation'], + 'text/x-c': ['c-lang'], + 'application/x-executable': ['elf'], + 'video/x-matroska': ['matroska'], + 'application/x-rpm': ['rpm'], + 'application/vnd.oasis.opendocument.image-template': ['archive'], + 'application/x-gnupg-keyring': ['gnu'], + 'application/x-compress': ['compress'], + 'image/x-quicktime': ['animation'], + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['msooxml'], + 'application/x-elc': ['lisp'], + 'application/x-epoc-app': ['epoc'], + 'application/vnd.oasis.opendocument.presentation-template': ['archive'], + 'image/x-pcx': ['images'], + 'application/x-epoc-word': ['epoc'], + 'application/vnd.font-fontforge-sfd': ['fonts'], + 'audio/x-w64': ['riff'], + 'application/x-dosexec': ['msdos'], + 'application/x-sc': ['sc'], + 'text/html': ['sgml'], + 'image/x-epoc-sketch': ['epoc'], + 'application/x-tokyocabinet-fixed': ['database'], + 'application/x-coredump': ['elf'], + 'video/3gpp2': ['animation'], + 'application/x-eet': ['archive'], + 'application/x-xz': ['compress'], + 'application/x-arc': ['archive'], + 'text/rtf': ['rtf'], + 'image/png': ['images'], + 'video/mp2t': ['animation'], + 'image/x-dpx': ['images'], + 'application/zip': ['archive', 'msdos'], + 'application/x-tokyocabinet-hash': ['database'], + 'video/x-flv': ['flash'], + 'text/x-pascal': ['pascal'], + 'application/x-epoc-opo': ['epoc'], + 'application/x-epoc-agenda': ['epoc'], + 'application/vnd.oasis.opendocument.chart': ['archive'], + 'audio/x-flac': ['audio'], + 'application/x-epoc-sheet': ['epoc'], + 'audio/vnd.dolby.dd-raw': ['dolby'], + 'application/octet-stream': ['archive', 'elf', 'compress'], + 'image/x-lss16': ['linux'], + 'application/x-lrzip': ['compress'], + 'application/x-object': ['elf'], + 'application/x-quark-xpress-3': ['wordprocessors'], + 'text/x-po': ['gnu'], + 'rinex/broadcast': ['rinex'], + 'image/x-award-bioslogo': ['images'], + 'message/rfc822': ['mail.news'], + 'application/vnd.ms-cab-compressed': ['msdos'], + 'video/x-flc': ['animation'], + 'text/x-bcpl': ['c-lang'], + 'video/mpeg4-generic': ['animation'], + 'application/x-zoo': ['archive'], + 'text/x-m4': ['m4'], + 'application/vnd.oasis.opendocument.spreadsheet-template': ['archive'], + 'text/x-ruby': ['ruby'], + 'application/x-7z-compressed': ['compress'], + 'application/x-pgp-keyring': ['pgp'], + 'application/ogg': ['vorbis'], + 'text/x-tcl': ['tcl'], + 'text/x-tex': ['tex'], + 'text/x-shellscript': ['commands'], + 'audio/x-hx-aac-adts': ['animation'], + 'image/x-ms-bmp': ['images'], + 'image/x-x3f': ['images'], + 'image/jpm': ['jpeg'], + 'application/x-setupscript': ['windows'], + 'application/vnd.iccprofile': ['icc'], + 'text/x-gawk': ['commands'], + 'image/x-niff': ['images'], + 'application/vnd.oasis.opendocument.formula-template': ['archive'], + 'application/vnd.ms-fontobject': ['fonts'], + 'audio/x-ape': ['audio'], + 'application/x-ichitaro5': ['wordprocessors'], + 'application/vnd.oasis.opendocument.text-template': ['archive'], + 'application/vnd.oasis.opendocument.spreadsheet': ['archive'], + 'application/vnd.lotus-wordpro': ['msdos'], + 'application/vnd.oasis.opendocument.formula': ['archive'], + 'application/vnd.google-earth.kml+xml': ['kml'], + 'image/jpeg': ['jpeg'], + 'rinex/navigation': ['rinex'], + 'application/x-debian-package': ['archive'], + 'text/x-info': ['tex'], + 'text/x-nawk': ['commands'], + 'text/troff': ['troff'], + 'text/x-java': ['java'], + 'application/epub+zip': ['archive'], + 'message/news': ['mail.news'], + 'application/x-java-jce-keystore': ['java'], + 'application/marc': ['marc21'], + 'text/x-msdos-batch': ['msdos'], + 'image/tiff': ['images'], + 'video/mpv': ['animation'], + 'application/x-svr4-package': ['pkgadd'], + 'application/vnd.oasis.opendocument.graphics-template': ['archive'], + 'video/x-mng': ['animation'], + 'text/calendar': ['misctools'], + 'application/gzip': ['compress'], + 'text/x-diff': ['diff'], + 'application/vnd.ms-tnef': ['mail.news', 'msdos'], + 'application/vnd.ms-opentype': ['fonts'], + 'application/x-archive': ['archive'], + 'application/vnd.ms-excel': ['msdos'], + 'application/x-tex-tfm': ['tex'], + 'text/x-awk': ['commands'], + 'application/x-lharc': ['archive'], + 'audio/x-hx-aac-adif': ['animation'], + 'image/x-canon-crw': ['images'], + 'application/x-arj': ['archive'], + 'application/vnd.google-earth.kmz': ['kml'], + 'application/x-java-keystore': ['java'], + 'application/vnd.rn-realmedia': ['audio'], + 'image/x-epoc-mbm': ['epoc'], + 'rinex/observation': ['rinex'], + 'application/x-sharedlib': ['elf'], + 'image/x-exr': ['images'], + 'x-epoc/x-sisx-app': ['archive'], + 'application/x-ima': ['filesystems'], + 'application/x-ia-arc': ['warc'], + 'application/msword': ['msdos'], + 'application/x-gnucash': ['sgml'], + 'application/x-lzip': ['compress'], + 'image/x-cpi': ['images'], + 'application/x-rar': ['archive'], + 'application/pgp-keys': ['pgp'], + 'video/x-fli': ['animation'], + 'application/x-tar': ['archive'], + 'video/x-msvideo': ['riff'], + 'image/vnd.adobe.photoshop': ['images'], + 'image/x-portable-pixmap': ['images'], + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['msooxml'], + 'application/x-hdf': ['images'], + 'image/x-xpmi': ['images'], + 'application/vnd.cups-raster': ['cups'], + 'video/h264': ['animation'], + 'application/x-shockwave-flash': ['flash'], + 'audio/x-mp4a-latm': ['animation'], + 'image/x-xwindowdump': ['images'], + 'audio/x-wav': ['riff'], + 'application/vnd.openxmlformats-officedocument.presentationml.presentation': ['msooxml'], + 'application/x-gdbm': ['database'], + 'application/vnd.tcpdump.pcap': ['sniffer'], + 'image/vnd.djvu': ['images'], + 'video/mp2p': ['animation'], + 'text/x-lisp': ['lisp'], + 'audio/mpeg': ['animation'], + 'application/x-tokyocabinet-table': ['database'], + 'audio/x-mod': ['audio'], + 'image/x-olympus-orf': ['images'], + 'application/vnd.symbian.install': ['archive'], + 'video/mpeg': ['animation'], + 'application/x-ms-reader': ['msdos'], + 'application/x-dbm': ['database'], + 'video/mj2': ['jpeg'], + 'text/x-xmcd': ['kde'], + 'application/x-hwp': ['wordprocessors'], + 'application/javascript': ['javascript'], + 'rinex/meteorological': ['rinex'], + 'rinex/clock': ['rinex'], + 'video/mp4v-es': ['animation'], + 'application/vnd.oasis.opendocument.presentation': ['archive'], + 'application/x-lzma': ['compress'], + 'text/x-php': ['commands'], + 'video/x-jng': ['animation']} + def from_buffer(self, buf): """ Identify the contents of `buf` diff --git a/util/fileMimes b/util/fileMimes new file mode 100644 index 0000000..3a061a7 --- /dev/null +++ b/util/fileMimes @@ -0,0 +1,619 @@ +./warc application/warc +./warc:!:mime application/x-ia-arc +./animation:!:mime video/x-sgi-movie +./animation:!:mime video/quicktime +./animation:!:mime video/quicktime +./animation:#!:mime video/quicktime +./animation:#!:mime video/quicktime +./animation:#!:mime video/quicktime +./animation:!:mime image/x-quicktime +./animation:#!:mime image/x-quicktime +./animation:!:mime application/x-quicktime-player +./animation:!:mime image/jp2 +./animation:!:mime video/mp4 +./animation:!:mime video/mp4 +./animation:!:mime video/mp4 +./animation:!:mime image/jp2 +./animation:!:mime video/3gpp +./animation:!:mime video/3gpp +./animation:!:mime video/3gpp +./animation:!:mime video/3gpp +./animation:!:mime video/3gpp2 +./animation:!:mime video/mp4 +./animation:!:mime video/3gpp +./animation:!:mime audio/mp4 +./animation:!:mime video/mp4 +./animation:!:mime video/quicktime +./animation:!:mime video/mpeg +./animation:!:mime video/mpeg4-generic +./animation:!:mime video/mpeg4-generic +./animation:!:mime video/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/mpeg +./animation:!:mime audio/x-hx-aac-adif +./animation:!:mime audio/x-hx-aac-adts +./animation:!:mime audio/x-mp4a-latm +./animation:#!:mime audio/x-mp4a-latm +./animation:!:mime video/x-fli +./animation:!:mime video/x-flc +./animation:#!:mime video/x-unknown +./animation:#!:mime video/x-unknown +./animation:!:mime video/x-ms-asf +./animation:!:mime video/x-mng +./animation:!:mime video/x-jng +./animation:!:mime model/vrml +./animation:!:mime model/vrml +./animation:!:mime model/x3d +./animation:# FIXME: This section is from the old magic.mime file and needs integrating with the rest +./animation:!:mime video/mp2p +./animation:!:mime video/mpeg +./animation:!:mime video/mpeg +./animation:!:mime video/mp4v-es +./animation:!:mime video/mp4v-es +./animation:!:mime video/mpv +./animation:!:mime video/mp2t +./animation:!:mime video/h264 +./archive:!:mime application/x-tar # encoding: posix +./archive:!:mime application/x-tar # encoding: gnu +./archive:!:mime application/x-cpio +./archive:!:mime application/x-cpio # encoding: swapped +./archive:!:mime application/x-archive +./archive:!:mime application/x-debian-package +./archive:!:mime application/x-archive +./archive:!:mime application/x-archive +./archive:!:mime application/x-arc +./archive:!:mime application/x-arc +./archive:!:mime application/x-arc +./archive:!:mime application/x-arc +./archive:!:mime application/x-arc +./archive:!:mime application/x-arc +./archive:!:mime application/x-arc +./archive:!:mime application/x-arc +./archive:!:mime application/x-arc +./archive:!:mime application/x-arj +./archive:!:mime application/x-lharc +./archive:!:mime application/x-lharc +./archive:!:mime application/x-lharc +./archive:!:mime application/x-lharc +./archive:!:mime application/x-lha +./archive:!:mime application/x-lha +./archive:!:mime application/x-lha +./archive:!:mime application/x-lha +./archive:!:mime application/x-lha +./archive:!:mime application/x-lha +./archive:!:mime application/x-lha +./archive:!:mime application/x-lha +./archive:!:mime application/x-lha +./archive:!:mime application/x-rar +./archive:!:mime application/zip +./archive:# Specialised zip formats which start with a member named 'mimetype' +./archive:# Check for have 8-byte name, 0-byte extra field, name "mimetype", and +./archive:>26 string \x8\0\0\0mimetypeapplication/ +./archive:# (mimetype contains "application/vnd.kde.") +./archive:# (mimetype contains "application/vnd.sun.xml.") +./archive:# (mimetype contains "application/vnd.oasis.opendocument.") +./archive:!:mime application/vnd.oasis.opendocument.text +./archive:!:mime application/vnd.oasis.opendocument.text-template +./archive:!:mime application/vnd.oasis.opendocument.text-web +./archive:!:mime application/vnd.oasis.opendocument.text-master +./archive:!:mime application/vnd.oasis.opendocument.graphics +./archive:!:mime application/vnd.oasis.opendocument.graphics-template +./archive:!:mime application/vnd.oasis.opendocument.presentation +./archive:!:mime application/vnd.oasis.opendocument.presentation-template +./archive:!:mime application/vnd.oasis.opendocument.spreadsheet +./archive:!:mime application/vnd.oasis.opendocument.spreadsheet-template +./archive:!:mime application/vnd.oasis.opendocument.chart +./archive:!:mime application/vnd.oasis.opendocument.chart-template +./archive:!:mime application/vnd.oasis.opendocument.formula +./archive:!:mime application/vnd.oasis.opendocument.formula-template +./archive:!:mime application/vnd.oasis.opendocument.database +./archive:!:mime application/vnd.oasis.opendocument.image +./archive:!:mime application/vnd.oasis.opendocument.image-template +./archive:>0x1E string mimetypeapplication/epub+zip EPUB document +./archive:!:mime application/epub+zip +./archive:# Catch other ZIP-with-mimetype formats +./archive:# always "PK". The 2 regex rules here print the "mimetype" member's +./archive:# (mimetype contains "application/") +./archive:!:mime application/zip +./archive:# (mimetype contents other than "application/*") +./archive:>26 string \x8\0\0\0mimetype +./archive:!:mime application/zip +./archive:!:mime application/jar +./archive:>>26 string !\x8\0\0\0mimetype Zip archive data +./archive:!:mime application/zip +./archive:!:mime application/x-zoo +./archive:!:mime application/octet-stream +./archive:!:mime application/x-bittorrent +./archive:!:mime application/x-eet +./archive:!:mime application/vnd.symbian.install +./archive:!:mime x-epoc/x-sisx-app +./archive:# archive must be an uncompressed file called 'mimetype' with contents +./iff:!:mime audio/x-aiff +./iff:!:mime audio/x-aiff +./iff:!:mime audio/x-aiff +./rinex:!:mime rinex/broadcast +./rinex:!:mime rinex/observation +./rinex:!:mime rinex/clock +./rinex:!:mime rinex/navigation +./rinex:!:mime rinex/navigation +./rinex:!:mime rinex/navigation +./rinex:!:mime rinex/meteorological +./rinex:!:mime rinex/navigation +./rinex:!:mime rinex/observation +./printer:!:mime application/postscript +./printer:!:mime application/postscript +./ruby:!:mime text/x-ruby +./ruby:!:mime text/x-ruby +./ruby:!:mime text/x-ruby +./ruby:!:mime text/x-ruby +./ruby:!:mime text/x-ruby +./ruby:!:mime text/x-ruby +./images:!:mime image/x-portable-bitmap +./images:!:mime image/x-portable-greymap +./images:!:mime image/x-portable-pixmap +./images:!:mime image/x-portable-bitmap +./images:!:mime image/x-portable-greymap +./images:!:mime image/x-portable-pixmap +./images:!:mime image/x-portable-pixmap +./images:!:mime image/x-niff +./images:!:mime image/x-canon-crw +./images:!:mime image/x-canon-cr2 +./images:!:mime image/tiff +./images:!:mime image/tiff +./images:!:mime image/tiff +./images:!:mime image/tiff +./images:!:mime image/png +./images:!:mime image/x-unknown +./images:!:mime image/x-unknown +./images:!:mime image/x-unknown +./images:!:mime image/gif +./images:!:mime image/x-award-bioslogo +./images:!:mime image/x-award-bioslogo +./images:!:mime image/x-award-bmp +./images:!:mime image/x-ms-bmp +./images:!:mime image/x-ms-bmp +./images:!:mime image/x-ms-bmp +./images:!:mime image/x-ms-bmp +./images:!:mime image/x-xpmi +./images:!:mime application/dicom +./images:!:mime image/x-xwindowdump +./images:!:mime image/x-pcx +./images:#!:mime image/pcx +./images:!:mime image/vnd.adobe.photoshop +./images:!:mime image/vnd.djvu +./images:!:mime image/vnd.djvu +./images:!:mime image/vnd.djvu +./images:!:mime image/vnd.djvu +./images:!:mime image/x-exr +./images:!:mime image/x-dpx +./images:!:mime application/x-hdf +./images:!:mime application/x-hdf +./images:!:mime image/x-cpi +./images:!:mime image/x-polar-monitor-bitmap +./images:# URL: http://webcvs.freedesktop.org/mime/shared-mime-info/freedesktop.org.xml.in?view=markup +./images:!:mime image/x-olympus-orf +./images:!:mime image/x-olympus-orf +./images:!:mime image/x-olympus-orf +./images:#!mime image/vnd.radiance +./images:#!mime image/x-pfs +./images:!:mime image/x-x3f +./images:!:mime image/x-paintnet +./pdf:!:mime application/pdf +./rtf:!:mime text/rtf +./tcl:!:mime text/x-lua +./tcl:!:mime text/x-tcl +./tcl:!:mime text/x-tcl +./tcl:!:mime text/x-tcl +./tcl:!:mime text/x-tcl +./tcl:!:mime text/x-tcl +./tcl:!:mime text/x-tcl +./tcl:!:mime text/x-tcl +./gnumeric:!:mime application/x-gnumeric +./cups:!:mime application/vnd.cups-raster +./cups:!:mime application/vnd.cups-raster +./riff:!:mime audio/x-wav +./riff:!:mime image/x-coreldraw +./riff:!:mime video/x-msvideo +./riff:!:mime audio/x-w64 +./riff:!:mime audio/x-wav +./make:!:mime text/x-makefile +./make:!:mime text/x-makefile +./make:!:mime text/x-makefile +./make:!:mime text/x-makefile +./make:!:mime text/x-makefile +./windows:!:mime application/x-setupscript. +./linux:# syslinux-4.05/mime/image/x-lss16.xml +./linux:!:mime image/x-lss16 +./lua:!:mime text/x-lua +./lua:!:mime text/x-lua +./lua:!:mime text/x-lua +./lua:!:mime text/x-lua +./gimp:!:mime image/x-xcf +./lisp:!:mime text/x-lisp +./lisp:!:mime text/x-lisp +./lisp:!:mime text/x-lisp +./lisp:!:mime text/x-lisp +./lisp:!:mime text/x-lisp +./lisp:!:mime text/x-lisp +./lisp:!:mime application/x-elc +./lisp:!:mime application/x-elc +./lisp:!:mime text/texmacs +./dolby:!:mime audio/vnd.dolby.dd-raw +./flash:!:mime application/x-shockwave-flash +./flash:!:mime application/x-shockwave-flash +./flash:!:mime video/x-flv +./msooxml:# Since MSOOXML doesn't have anything like the uncompressed "mimetype" +./msooxml:# Correct the mimetype with the registered ones: +./msooxml:!:mime application/vnd.openxmlformats-officedocument.wordprocessingml.document +./msooxml:!:mime application/vnd.openxmlformats-officedocument.presentationml.presentation +./msooxml:!:mime application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +./filesystems:!:mime application/x-ima +./filesystems:!:mime application/x-iso9660-image +./filesystems:!:mime application/x-iso9660-image +./pgp:!:mime application/x-pgp-keyring +./pgp:!:mime application/x-pgp-keyring +./pgp:!:mime application/x-pgp-keyring +./pgp:#!:mime application/pgp-encrypted +./pgp:!:mime text/PGP # encoding: armored data +./pgp:!:mime application/pgp-keys +./pgp:!:mime application/pgp +./pgp:!:mime application/pgp-signature +./m4:!:mime text/x-m4 +./epoc:!:mime image/x-epoc-mbm +./epoc:!:mime image/x-epoc-sketch +./epoc:!:mime application/x-epoc-word +./epoc:!:mime application/x-epoc-opl +./epoc:!:mime application/x-epoc-sheet +./epoc:!:mime application/x-epoc-opo +./epoc:!:mime application/x-epoc-app +./epoc:!:mime application/x-epoc-agenda +./epoc:!:mime application/x-epoc-data +./epoc:!:mime application/x-epoc-jotter +./tex:!:mime application/x-dvi +./tex:!:mime application/x-tex-tfm +./tex:!:mime application/x-tex-tfm +./tex:!:mime text/x-texinfo +./tex:!:mime text/x-info +./tex:!:mime text/x-tex +./tex:!:mime text/x-tex +./tex:!:mime text/x-tex +./tex:!:mime text/x-tex +./tex:!:mime text/x-tex +./tex:!:mime text/x-tex +./tex:!:mime text/x-tex +./tex:!:mime text/x-tex +./tex:!:mime text/x-tex +./elf:!:mime application/octet-stream +./elf:!:mime application/x-object +./elf:!:mime application/x-executable +./elf:!:mime application/x-sharedlib +./elf:!:mime application/x-coredump +./sc:!:mime application/x-sc +./javascript:!:mime application/javascript +./javascript:!:mime application/javascript +./javascript:!:mime application/javascript +./javascript:!:mime application/javascript +./javascript:!:mime application/javascript +./javascript:!:mime application/javascript +./assembler:!:mime text/x-asm +./assembler:!:mime text/x-asm +./assembler:!:mime text/x-asm +./assembler:!:mime text/x-asm +./assembler:!:mime text/x-asm +./assembler:!:mime text/x-asm +./assembler:!:mime text/x-asm +./scientific:# http://www.ch.ic.ac.uk/chemime/ +./scientific:# e.g., the example on the chemime site. +./scientific:!:mime chemical/x-pdb +./graphviz:#!:mime text/vnd.graphviz +./graphviz:#!:mime text/vnd.graphviz +./mail.news:!:mime message/rfc822 +./mail.news:!:mime message/rfc822 +./mail.news:!:mime message/rfc822 +./mail.news:!:mime message/rfc822 +./mail.news:!:mime message/rfc822 +./mail.news:!:mime message/rfc822 +./mail.news:!:mime message/rfc822 +./mail.news:!:mime message/news +./mail.news:!:mime message/news +./mail.news:!:mime message/rfc822 +./mail.news:!:mime message/news +./mail.news:!:mime message/rfc822 +./mail.news:!:mime application/vnd.ms-tnef +./compress:!:mime application/x-compress +./compress:!:mime application/gzip +./compress:!:mime application/octet-stream +./compress:!:mime application/octet-stream +./compress:!:mime application/octet-stream +./compress:!:mime application/octet-stream +./compress:!:mime application/octet-stream +./compress:!:mime application/x-bzip2 +./compress:!:mime application/x-lzip +./compress:!:mime application/x-7z-compressed +./compress:!:mime application/x-lzma +./compress:!:mime application/x-xz +./compress:!:mime application/x-lrzip +./mime:# $File: mime,v 1.6 2010/11/25 15:00:12 christos Exp $ +./mime:# mime: file(1) magic for MIME encoded files +./matroska:!:mime video/webm +./matroska:!:mime video/x-matroska +./frame:!:mime application/x-mif +./frame:!:mime application/x-mif +./frame:!:mime application/x-mif +./frame:!:mime application/x-mif +./frame:!:mime application/x-mif +./frame:!:mime application/x-mif +./frame:#!:mime application/x-mif +./frame:!:mime application/x-mif +./cafebabe:!:mime application/x-java-applet +./cafebabe:!:mime application/x-java-pack200 +./cafebabe:!:mime application/x-java-pack200 +./pkgadd:!:mime application/x-svr4-package +./rpm:!:mime application/x-rpm +./rpm:!:mime application/x-rpm +./macintosh:!:mime application/mac-binhex40 +./macintosh:!:mime application/x-stuffit +./macintosh:!:mime application/x-stuffit +./kml:!:mime application/vnd.google-earth.kml+xml +./kml:!:mime application/vnd.google-earth.kml+xml +./kml:!:mime application/vnd.google-earth.kmz +./python:!:mime text/x-python +./python:!:mime text/x-python +./python:!:mime text/x-python +./python:!:mime text/x-python +./python:!:mime text/x-python +./python:!:mime text/x-python +./python:!:mime text/x-python +./python:!:mime text/x-python +./python:!:mime text/x-python +./python:!:mime text/x-python +./python:!:mime text/x-python +./misctools:!:mime text/calendar +./misctools:!:mime text/x-vcard +./xwindows:!:mime image/x-xcursor +./icc:!:mime application/vnd.iccprofile +./icc:!:mime application/vnd.iccprofile +./icc:!:mime application/vnd.iccprofile +./icc:!:mime application/vnd.iccprofile +./icc:!:mime application/vnd.iccprofile +./kde:!:mime application/x-kdelnk +./kde:!:mime application/x-kdelnk +./kde:!:mime text/x-xmcd +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-nawk +./commands:!:mime text/x-nawk +./commands:!:mime text/x-nawk +./commands:!:mime text/x-gawk +./commands:!:mime text/x-gawk +./commands:!:mime text/x-gawk +./commands:!:mime text/x-awk +./commands:!:mime text/x-awk +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-shellscript +./commands:!:mime text/x-php +./commands:!:mime text/x-php +./commands:!:mime text/x-php +./commands:!:mime text/x-php +./commands:!:mime text/x-php +./commands:!:mime text/x-php +./c-lang:!:mime text/x-bcpl +./c-lang:!:mime text/x-bcpl +./c-lang:!:mime text/x-c +./c-lang:!:mime text/x-c +./c-lang:!:mime text/x-c +./c-lang:!:mime text/x-c +./c-lang:!:mime text/x-c +./c-lang:!:mime text/x-c +./c-lang:!:mime text/x-c +./c-lang:!:mime text/x-c +./c-lang:!:mime text/x-c++ +./c-lang:!:mime text/x-c++ +./c-lang:!:mime text/x-c++ +./c-lang:!:mime text/x-c++ +./c-lang:!:mime text/x-c++ +./fonts:!:mime application/x-font-sfn +./fonts:!:mime application/x-font-sfn +./fonts:!:mime application/x-font-ttf +./fonts:!:mime application/vnd.ms-opentype +./fonts:!:mime application/vnd.font-fontforge-sfd +./fonts:!:mime application/vnd.ms-fontobject +./msdos:!:mime text/x-msdos-batch +./msdos:!:mime text/x-msdos-batch +./msdos:!:mime text/x-msdos-batch +./msdos:!:mime text/x-msdos-batch +./msdos:!:mime application/x-dosexec +./msdos:!:mime application/zip +./msdos:!:mime application/zip +./msdos:!:mime application/x-lha +./msdos:!:mime application/x-lha +./msdos:!:mime application/msword +./msdos:!:mime application/msword +./msdos:!:mime application/msword +./msdos:!:mime application/msword +./msdos:!:mime application/msword +./msdos:!:mime application/msword +./msdos:!:mime application/msword +./msdos:!:mime application/msword +./msdos:!:mime application/msword +./msdos:!:mime application/vnd.ms-excel +./msdos:!:mime application/msword +./msdos:!:mime application/vnd.ms-excel +./msdos:!:mime application/vnd.ms-excel +./msdos:!:mime application/vnd.ms-excel +./msdos:!:mime application/vnd.ms-excel +./msdos:!:mime application/x-123 +./msdos:!:mime application/x-123 +./msdos:!:mime application/vnd.lotus-wordpro +./msdos:!:mime application/vnd.lotus-wordpro +./msdos:!:mime image/x-icon +./msdos:!:mime application/x-dosexec +./msdos:!:mime application/vnd.ms-tnef +./msdos:!:mime application/vnd.ms-cab-compressed +./msdos:#!:mime application/vnd.ms-excel +./msdos:!:mime application/msword +./msdos:!:mime application/msword +./msdos:!:mime application/msword +./msdos:!:mime application/msword +./msdos:!:mime application/x-ms-reader +./troff:!:mime text/troff +./troff:!:mime text/troff +./troff:!:mime text/troff +./troff:!:mime text/troff +./troff:!:mime text/troff +./troff:!:mime text/troff +./troff:!:mime text/troff +./pascal:!:mime text/x-pascal +./pascal:!:mime text/x-pascal +./pascal:!:mime text/x-pascal +./gnu:# Note: magic.mime had 0x8501 for the next line instead of 0x8502 +./gnu:!:mime text/PGP # encoding: data +./gnu:!:mime application/x-gnupg-keyring +./gnu:!:mime text/x-po +./diff:!:mime text/x-diff +./diff:!:mime text/x-diff +./diff:!:mime text/x-diff +./diff:!:mime text/x-diff +./diff:!:mime text/x-diff +./diff:!:mime text/x-diff +./jpeg:!:mime image/jpeg +./jpeg:# Added sub-entries for JP2, JPX, JPM and MJ2 formats; added mimetypes +./jpeg:!:mime image/jp2 +./jpeg:!:mime image/jpx +./jpeg:!:mime image/jpm +./jpeg:!:mime video/mj2 +./vorbis:!:mime application/ogg +./marc21:!:mime application/marc +./marc21:!:mime application/marc +./marc21:!:mime application/marc +./marc21:!:mime application/marc +./marc21:!:mime application/marc +./marc21:!:mime application/marc +./perl:!:mime text/x-perl +./perl:!:mime text/x-perl +./perl:!:mime text/x-perl +./perl:!:mime text/x-perl +./perl:!:mime text/x-perl +./perl:!:mime text/x-perl +./perl:!:mime text/x-perl +./perl:!:mime text/x-perl +./perl:!:mime text/x-perl +./fortran:!:mime text/x-fortran +./sniffer:!:mime application/vnd.tcpdump.pcap +./sniffer:!:mime application/vnd.tcpdump.pcap +./adventure:#!:mime application/x-adrift +./database:!:mime application/x-gdbm +./database:!:mime application/x-gdbm +./database:!:mime application/x-gdbm +./database:!:mime application/x-dbm +./database:!:mime application/x-dbf +./database:#!:mime application/x-dbf; charset=unknown-8bit ?? +./database:#!:mime application/x-dbase +./database:#!:mime application/x-mdx +./database:!:mime application/x-msaccess +./database:!:mime application/x-msaccess +./database:!:mime application/x-tokyocabinet-hash +./database:!:mime application/x-tokyocabinet-btree +./database:!:mime application/x-tokyocabinet-fixed +./database:!:mime application/x-tokyocabinet-table +./wordprocessors:!:mime application/x-hwp +./wordprocessors:!:mime application/x-quark-xpress-3 +./wordprocessors:!:mime application/x-ichitaro4 +./wordprocessors:!:mime application/x-ichitaro5 +./wordprocessors:!:mime application/x-ichitaro6 +./wordprocessors:!:mime application/x-freemind +./wordprocessors:!:mime application/x-scribus +./sgml:!:mime image/svg+xml +./sgml:!:mime application/x-gnucash +./sgml:!:mime application/xml-sitemap +./sgml:!:mime text/html +./sgml:!:mime text/html +./sgml:!:mime text/html +./sgml:!:mime text/html +./sgml:!:mime text/html +./sgml:!:mime text/html +./sgml:!:mime text/html +./sgml:!:mime text/html +./sgml:!:mime text/html +./sgml:!:mime text/html +./sgml:!:mime text/html +./sgml:!:mime application/xml +./sgml:!:mime application/xml +./sgml:!:mime application/xml +./sgml:!:mime application/xml +./sgml:!:mime application/xml +./sgml:!:mime application/xml +./audio:!:mime audio/basic +./audio:!:mime audio/basic +./audio:!:mime audio/basic +./audio:!:mime audio/basic +./audio:!:mime audio/basic +./audio:!:mime audio/basic +./audio:!:mime audio/basic +./audio:!:mime audio/x-adpcm +./audio:!:mime audio/x-dec-basic +./audio:!:mime audio/x-dec-basic +./audio:!:mime audio/x-dec-basic +./audio:!:mime audio/x-dec-basic +./audio:!:mime audio/x-dec-basic +./audio:!:mime audio/x-dec-basic +./audio:!:mime audio/x-dec-basic +./audio:!:mime audio/x-dec-basic +./audio:!:mime audio/midi +./audio:!:mime audio/x-unknown +./audio:!:mime audio/x-unknown +./audio:!:mime audio/x-unknown +./audio:!:mime audio/x-pn-realaudio +./audio:!:mime application/vnd.rn-realmedia +./audio:# sigh, there are many mimes for that but the above are the most common. +./audio:# mime types according to http://www.geocities.com/nevilo/mod.htm: +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-mod +./audio:!:mime audio/x-flac +./audio:!:mime audio/x-ape +./audio:!:mime audio/x-musepack +./java:!:mime application/x-java-keystore +./java:!:mime application/x-java-jce-keystore +./java:!:mime text/x-java diff --git a/util/parse_magdir.py b/util/parse_magdir.py new file mode 100644 index 0000000..6f32cca --- /dev/null +++ b/util/parse_magdir.py @@ -0,0 +1,52 @@ +''' +Generate fileMimes as: +cd libmagic/magic/Magicdir +grep -R mime . > fileMimes + +Then run this script to generate file_types.py +It's a simple dictionary of all mime-types and what Magicdir file +detects them. +''' + +lines = open('fileMimes', 'r').readlines() + +def dedupe(fileMimes): + mimeType = [] + source = [] + final = {} + for item in fileMimes: + try: + index = source[mimeType.index(item[0])] + try: + index.index(item[1]) + except: + index.append(item[1]) + except: + mimeType.append(item[0]) + source.append([item[1]]) + for item in mimeType: + final[item] = source[mimeType.index(item)] + + return final + + + +file_types = [] + +for line in lines: + try: + temp = line.split(':!:mime') + temp[0] = temp[0].lstrip('./') + temp[1] = temp[1].lstrip().rstrip() + if temp[1][0] != '#': + try: + temp[1] = temp[1].split('#')[0].rstrip().rstrip('.') + except: + pass + file_types.append((temp[1], temp[0])) + except: + pass + +file_types = dedupe(file_types) +file_types = ("],\n" + ' ' * 12 + "'").join(str(file_types).split("], '")) +open('file_types.py', 'w').write(' '*8 + 'self.file_types = ' + file_types)