From 21559e9f4bd58b1504cc0fda7b0dceee0c833862 Mon Sep 17 00:00:00 2001 From: zyp Date: Thu, 26 Jan 2006 19:07:15 +0000 Subject: [project @ zyp-20060126190715-557e941315671b81] [project @ 18] Moved disccat and pyanidb to the right location. Added pyqtmpc and ophidia. --- anidb.py | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++ anidb_add.py | 85 +++++++++++++++++++++++++++++++++++++++++++++ ed2k/Makefile | 27 +++++++++++++++ ed2k/__init__.py | 11 ++++++ ed2k/ed2k.cpp | 68 ++++++++++++++++++++++++++++++++++++ ed2k/ed2k.h | 19 +++++++++++ ed2k/ed2k_wrapper.cpp | 10 ++++++ hash/Makefile | 27 +++++++++++++++ hash/__init__.py | 11 ++++++ hash/crc32.cpp | 41 ++++++++++++++++++++++ hash/crc32.h | 11 ++++++ hash/ed2k.cpp | 41 ++++++++++++++++++++++ hash/ed2k.h | 17 +++++++++ hash/hash.cpp | 85 +++++++++++++++++++++++++++++++++++++++++++++ hash/hash.h | 37 ++++++++++++++++++++ hash/hash_wrapper.cpp | 14 ++++++++ pyanidb.conf | 5 +++ 17 files changed, 604 insertions(+) create mode 100644 anidb.py create mode 100644 anidb_add.py create mode 100644 ed2k/Makefile create mode 100644 ed2k/__init__.py create mode 100644 ed2k/ed2k.cpp create mode 100644 ed2k/ed2k.h create mode 100644 ed2k/ed2k_wrapper.cpp create mode 100644 hash/Makefile create mode 100644 hash/__init__.py create mode 100644 hash/crc32.cpp create mode 100644 hash/crc32.h create mode 100644 hash/ed2k.cpp create mode 100644 hash/ed2k.h create mode 100644 hash/hash.cpp create mode 100644 hash/hash.h create mode 100644 hash/hash_wrapper.cpp create mode 100644 pyanidb.conf diff --git a/anidb.py b/anidb.py new file mode 100644 index 0000000..1a36ec3 --- /dev/null +++ b/anidb.py @@ -0,0 +1,95 @@ +import socket, time + +protover = 3 +client = 'pyanidb' +clientver = 2 + +states = { + 'unknown': 0, + 'hdd': 1, + 'cd': 2, + 'deleted': 3, + 'shared': 4, + 'release': 5 +} + +class AniDBError(Exception): + pass + +class AniDBTimeout(AniDBError): + pass + +class AniDBLoginError(AniDBError): + pass + +class AniDBUserError(AniDBLoginError): + pass + +class AniDBReplyError(AniDBError): + pass + +class AniDBUnknownFile(AniDBError): + pass + +class AniDB: + def __init__(self, username, password, localport = 1234, server = ('api.anidb.info', 9000)): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.sock.bind(('0.0.0.0', localport)) + self.sock.settimeout(10) + self.username = username.lower() + self.password = password + self.server = server + self.session = '' + self.lasttime = 0 + self.new_version = False + def __del__(self): + self.logout() + self.sock.close() + def execute(self, data): + t = time.time() + if t < self.lasttime + 2: + time.sleep(self.lasttime + 2 - t) + self.lasttime = time.time() + self.sock.sendto(data + '\n', self.server) + try: + data = self.sock.recv(8192).split('\n') + except socket.timeout: + raise AniDBTimeout() + code, text = data[0].split(' ', 1) + data = data[1:-1] + code = int(code) + return code, text, data + def ping(self): + t = time.time() + try: + return self.execute('PING')[0] == 300 and time.time() - t or None + except AniDBTimeout: + return None + def auth(self): + code, text, data = self.execute('AUTH user=%s&pass=%s&protover=%d&client=%s&clientver=%d' % (self.username, self.password, protover, client, clientver)) + if code in [200, 201]: + self.session = text.split(' ', 1)[0] + if code == 201: + self.new_version = True + elif code == 500: + raise AniDBUserError() + else: + raise AniDBReplyError(code, text) + def logout(self): + if self.session: + try: + self.execute('LOGOUT s=%s' % (self.session)) + self.session = '' + except AniDBError: + pass + def add_hash(self, size, ed2k, state = states['hdd'], viewed = False, source = '', storage = '', other = ''): + while 1: + code, text, data = self.execute('MYLISTADD s=%s&size=%d&ed2k=%s&state=%d&viewed=%d&source=%s&storage=%s&other=%s' % (self.session, size, ed2k, state, viewed and 1 or 0, source, storage, other)) + if code in [210, 310]: + return + elif code in [501, 506]: + self.auth() + elif code == 320: + raise AniDBUnknownFile() + else: + raise AniDBReplyError(code, text) diff --git a/anidb_add.py b/anidb_add.py new file mode 100644 index 0000000..c2fb1cf --- /dev/null +++ b/anidb_add.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python + +import ConfigParser, os, sys, thread, time, getpass, anidb, ed2k + +num_threads = 0 + +def hash_file(name): + if not os.access(name, os.R_OK): + print 'Invalid file: %s' % (name) + return + size = os.stat(name).st_size + hash = ed2k.file_hash(name) + print 'Hashed: ed2k://|file|%s|%d|%s|' % (name, size, hash) + return name, size, hash + +def hash_thread(filelist, hashlist): + global num_threads + num_threads += 1 + try: + while filelist: + h = hash_file(filelist.pop(0)) + if h: + hashlist.append(h) + except IndexError: + pass + num_threads -= 1 + +def auth(): + try: + c = ConfigParser.ConfigParser() + c.read(os.path.expanduser('~/.pyanidb.conf')) + username = c.get('auth', 'username') + password = c.get('auth', 'password') + except: + username = raw_input('Username: ') + password = getpass.getpass() + return username, password + +username, password = auth() + +try: + a = anidb.AniDB(username, password) + t = a.ping() + if t: + print 'AniDB is reachable, %.3fs' % (t) + else: + print 'AniDB is unreachable.' + sys.exit(1) + a.auth() + print 'Logged in as user %s.' % (username) + if a.new_version: + print 'New version available.' + + filelist = sys.argv[1:] + hashlist = [] + + thread.start_new_thread(hash_thread, (filelist, hashlist)) + + while hashlist or num_threads or filelist: + if not hashlist: + time.sleep(0.1) + continue + name, size, hash = hashlist.pop(0) + try: + while 1: + try: + a.add_hash(size, hash) + except anidb.AniDBTimeout: + print 'Connection timed out, retrying.' + continue + break + except anidb.AniDBUnknownFile: + print 'Unknown file: %s' % (name) + continue + print 'Added file: %s' % (name) + print 'All operations finished.' +except anidb.AniDBUserError: + print 'Invalid username/password.' + sys.exit(1) +except anidb.AniDBTimeout: + print 'Connection timed out.' + sys.exit(1) +except anidb.AniDBError, e: + print 'Fatal error:', e + sys.exit(1) diff --git a/ed2k/Makefile b/ed2k/Makefile new file mode 100644 index 0000000..07d23df --- /dev/null +++ b/ed2k/Makefile @@ -0,0 +1,27 @@ +CC=gcc +CFLAGS= +CPP=g++ +CPPFLAGS= +LD=gcc +LDFLAGS=-shared + +OBJECTS=ed2k_wrapper.o ed2k.o +TARGET=_ed2k.so +INCLUDE=-I /usr/include/python2.4/ +LIB=-l boost_python -l ssl + +all: $(TARGET) + +clean: + -rm $(TARGET) $(OBJECTS) + +# M�-regel +$(TARGET): $(OBJECTS) Makefile + $(LD) $(LDFLAGS) -o $(TARGET) $(OBJECTS) $(LIB) + +# Pseudoregler +%.o: %.c Makefile + $(CC) $(CFLAGS) -o $@ -c $< $(INCLUDE) + +%.o: %.cpp Makefile + $(CPP) $(CPPFLAGS) -o $@ -c $< $(INCLUDE) \ No newline at end of file diff --git a/ed2k/__init__.py b/ed2k/__init__.py new file mode 100644 index 0000000..754b5c9 --- /dev/null +++ b/ed2k/__init__.py @@ -0,0 +1,11 @@ +from _ed2k import * + +def file_hash(name): + e = Ed2k() + f = open(name) + data = f.read(32768) + while data: + e.update(data) + data = f.read(32768) + f.close() + return e.digest() diff --git a/ed2k/ed2k.cpp b/ed2k/ed2k.cpp new file mode 100644 index 0000000..369f57e --- /dev/null +++ b/ed2k/ed2k.cpp @@ -0,0 +1,68 @@ +#include "ed2k.h" + +namespace Hex { + static char* digits = "0123456789abcdef"; + std::string hex(char* bin, int length) { + std::string s(length * 2, ' '); + for(int i = 0; i < length; i++) { + s[i*2] = digits[(bin[i] >> 4) & 0xf]; + s[i*2+1] = digits[bin[i] & 0xf]; + } + return s; + } + std::string hex(int bin) { + std::string s(sizeof(int) * 2, ' '); + for(int i = 0; i < sizeof(int) * 2; i++) { + s[sizeof(int) * 2 - 1 - i] = digits[bin & 0xf]; + bin = bin >> 4; + } + return s; + } +} + +template +inline T min(T a, T b) { + return (a > b) ? b : a; +} + +Ed2k::Ed2k() { + MD4_Init(&md4_partial); + MD4_Init(&md4_final); + size_total = 0; + digest_str = ""; +} + +void Ed2k::update(std::string data_str) { + unsigned int length = data_str.length(); + const char* data = data_str.c_str(); + while(length) { + if(!(size_total % (9500 * 1024)) && size_total) { + unsigned char digest[16]; + MD4_Final(digest, &md4_partial); + MD4_Update(&md4_final, digest, 16); + MD4_Init(&md4_partial); + } + int size = min(length, (9500 * 1024) - (size_total % (9500 * 1024))); + MD4_Update(&md4_partial, data, size); + length -= size; + data += size; + size_total += size; + }; +} + +std::string Ed2k::digest() { + if(!digest_str.length()) { + char* digest = new char[16]; + if(size_total > (9500 * 1024)) { + unsigned char digest_partial[16]; + MD4_Final(digest_partial, &md4_partial); + MD4_Update(&md4_final, digest_partial, 16); + MD4_Final((unsigned char*)digest, &md4_final); + } else { + MD4_Final((unsigned char*)digest, &md4_partial); + } + digest_str = Hex::hex(digest, 16); + delete digest; + } + return digest_str; +} diff --git a/ed2k/ed2k.h b/ed2k/ed2k.h new file mode 100644 index 0000000..97feb61 --- /dev/null +++ b/ed2k/ed2k.h @@ -0,0 +1,19 @@ +#ifndef _ED2K_H_ +#define _ED2K_H_ + +#include +#include + +class Ed2k { + private: + MD4_CTX md4_partial; + MD4_CTX md4_final; + unsigned int size_total; + std::string digest_str; + public: + Ed2k(); + void update(std::string data); + std::string digest(); +}; + +#endif // _ED2K_H_ diff --git a/ed2k/ed2k_wrapper.cpp b/ed2k/ed2k_wrapper.cpp new file mode 100644 index 0000000..0a9f8c8 --- /dev/null +++ b/ed2k/ed2k_wrapper.cpp @@ -0,0 +1,10 @@ +#include "ed2k.h" + +#include +using namespace boost::python; + +BOOST_PYTHON_MODULE(_ed2k) { + class_("Ed2k") + .def("update", &Ed2k::update) + .def("digest", &Ed2k::digest); +} diff --git a/hash/Makefile b/hash/Makefile new file mode 100644 index 0000000..0d897d0 --- /dev/null +++ b/hash/Makefile @@ -0,0 +1,27 @@ +CC=gcc +CFLAGS= +CPP=g++ +CPPFLAGS= +LD=gcc +LDFLAGS=-shared + +OBJECTS=hash_wrapper.o hash.o crc32.o ed2k.o +TARGET=_hash.so +INCLUDE=-I /usr/include/python2.4/ +LIB=-l boost_python -l ssl + +all: $(TARGET) + +clean: + -rm $(TARGET) $(OBJECTS) + +# Mål-regel +$(TARGET): $(OBJECTS) Makefile + $(LD) $(LDFLAGS) -o $(TARGET) $(OBJECTS) $(LIB) + +# Pseudoregler +%.o: %.c Makefile + $(CC) $(CFLAGS) -o $@ -c $< $(INCLUDE) + +%.o: %.cpp Makefile + $(CPP) $(CPPFLAGS) -o $@ -c $< $(INCLUDE) \ No newline at end of file diff --git a/hash/__init__.py b/hash/__init__.py new file mode 100644 index 0000000..0fc1715 --- /dev/null +++ b/hash/__init__.py @@ -0,0 +1,11 @@ +from _hash import * + +def file_hash(name): + h = Hash() + f = open(name) + data = f.read(32768) + while data: + h.update(data) + data = f.read(32768) + f.close() + return h diff --git a/hash/crc32.cpp b/hash/crc32.cpp new file mode 100644 index 0000000..55d6dd2 --- /dev/null +++ b/hash/crc32.cpp @@ -0,0 +1,41 @@ +#include "crc32.h" + +int* CRC32::crc_table; + +int* CRC32::generate_table() { + int crc; + int* table = new int[256]; + for(int i = 0; i < 256; i++) { + crc = i << 24; + for(int j = 0; j < 8; j++) { + if(crc & 0x80000000) { + crc = (crc << 1) ^ 0x04c11db7; + } else { + crc = crc << 1; + } + } + table[i] = crc; + } + return table; +} + +int CRC32::reflect(int data, int bits) { + int x = 0; + for(int i = 0; i < bits; i++) { + x = x << 1; + x |= data & 1; + data = data >> 1; + } + return x; +} + +int CRC32::crc32(int crc, const char* data, int length) { + crc = ~reflect(crc, 32); + if(!crc_table) { + crc_table = generate_table(); + } + for (int i = 0; i < length; i++) { + crc = (crc << 8) ^ crc_table[((crc >> 24) ^ reflect(data[i], 8)) & 0xff]; + } + return ~reflect(crc, 32); +} diff --git a/hash/crc32.h b/hash/crc32.h new file mode 100644 index 0000000..646ccce --- /dev/null +++ b/hash/crc32.h @@ -0,0 +1,11 @@ +#ifndef _CRC32_H_ +#define _CRC32_H_ + +namespace CRC32 { + extern int* crc_table; + int* generate_table(); + int reflect(int data, int bits); + int crc32(int crc, const char* data, int length); +} + +#endif // _CRC32_H_ diff --git a/hash/ed2k.cpp b/hash/ed2k.cpp new file mode 100644 index 0000000..92e7b15 --- /dev/null +++ b/hash/ed2k.cpp @@ -0,0 +1,41 @@ +#include "ed2k.h" + +template +inline T min(T a, T b) { + return (a > b) ? b : a; +} + +Ed2k::Ed2k() { + MD4_Init(&md4_partial); + MD4_Init(&md4_final); + size_total = 0; +} + +void Ed2k::update(const char* data, int length) { + while(length) { + if(!(size_total % (9500 * 1024)) && size_total) { + unsigned char digest[16]; + MD4_Final(digest, &md4_partial); + MD4_Update(&md4_final, digest, 16); + MD4_Init(&md4_partial); + } + int size = min(length, (9500 * 1024) - (size_total % (9500 * 1024))); + MD4_Update(&md4_partial, data, size); + length -= size; + data += size; + size_total += size; + }; +} + +char* Ed2k::digest() { + char* digest = new char[16]; + if(size_total > (9500 * 1024)) { + unsigned char digest_partial[16]; + MD4_Final(digest_partial, &md4_partial); + MD4_Update(&md4_final, digest_partial, 16); + MD4_Final((unsigned char*)digest, &md4_final); + } else { + MD4_Final((unsigned char*)digest, &md4_partial); + } + return digest; +} diff --git a/hash/ed2k.h b/hash/ed2k.h new file mode 100644 index 0000000..1be7302 --- /dev/null +++ b/hash/ed2k.h @@ -0,0 +1,17 @@ +#ifndef _ED2K_H_ +#define _ED2K_H_ + +#include + +class Ed2k { + private: + MD4_CTX md4_partial; + MD4_CTX md4_final; + unsigned int size_total; + public: + Ed2k(); + void update(const char* data, int length); + char* digest(); +}; + +#endif // _ED2K_H_ diff --git a/hash/hash.cpp b/hash/hash.cpp new file mode 100644 index 0000000..d844c46 --- /dev/null +++ b/hash/hash.cpp @@ -0,0 +1,85 @@ +#include "hash.h" +#include "crc32.h" + +#include + +namespace Hex { + static char* digits = "0123456789abcdef"; + std::string hex(char* bin, int length) { + std::string s(length * 2, ' '); + for(int i = 0; i < length; i++) { + s[i*2] = digits[(bin[i] >> 4) & 0xf]; + s[i*2+1] = digits[bin[i] & 0xf]; + } + return s; + } + std::string hex(int bin) { + std::string s(sizeof(int) * 2, ' '); + for(int i = 0; i < sizeof(int) * 2; i++) { + s[sizeof(int) * 2 - 1 - i] = digits[bin & 0xf]; + bin = bin >> 4; + } + return s; + } +} + +Hash::Hash() { + finished = false; + + crc32_ctx = 0; + crc32_str = ""; + + ed2k_str = ""; + + MD5_Init(&md5_ctx); + md5_str = ""; + + SHA1_Init(&sha1_ctx); + sha1_str = ""; +} + +void Hash::update(std::string data) { + if(finished) { + throw std::runtime_error("Can't update after digest."); + } + crc32_ctx = CRC32::crc32(crc32_ctx, data.c_str(), data.length()); + ed2k_ctx.update(data.c_str(), data.length()); + MD5_Update(&md5_ctx, data.c_str(), data.length()); + SHA1_Update(&sha1_ctx, data.c_str(), data.length()); +} + +std::string Hash::crc32() { + return Hex::hex(crc32_ctx); +} + +std::string Hash::ed2k() { + if(!ed2k_str.length()) { + finished = true; + char* digest = ed2k_ctx.digest(); + ed2k_str = Hex::hex(digest, 16); + delete digest; + } + return ed2k_str; +} + +std::string Hash::md5() { + if(!md5_str.length()) { + finished = true; + char* digest = new char[16]; + MD5_Final((unsigned char*)digest, &md5_ctx); + md5_str = Hex::hex(digest, 16); + delete digest; + } + return md5_str; +} + +std::string Hash::sha1() { + if(!sha1_str.length()) { + finished = true; + char* digest = new char[20]; + SHA1_Final((unsigned char*)digest, &sha1_ctx); + sha1_str = Hex::hex(digest, 20); + delete digest; + } + return sha1_str; +} diff --git a/hash/hash.h b/hash/hash.h new file mode 100644 index 0000000..3c7b9c9 --- /dev/null +++ b/hash/hash.h @@ -0,0 +1,37 @@ +#ifndef _HASH_H_ +#define _HASH_H_ + +#include + +#include "ed2k.h" + +#include +#include +#include + +class Hash { + private: + bool finished; + + int crc32_ctx; + std::string crc32_str; + + Ed2k ed2k_ctx; + std::string ed2k_str; + + MD5_CTX md5_ctx; + std::string md5_str; + + SHA_CTX sha1_ctx; + std::string sha1_str; + + public: + Hash(); + void update(std::string data); + std::string crc32(); + std::string ed2k(); + std::string md5(); + std::string sha1(); +}; + +#endif // _HASH_H_ diff --git a/hash/hash_wrapper.cpp b/hash/hash_wrapper.cpp new file mode 100644 index 0000000..a4173b9 --- /dev/null +++ b/hash/hash_wrapper.cpp @@ -0,0 +1,14 @@ +#include "hash.h" + +#include +using namespace boost::python; + +BOOST_PYTHON_MODULE(_hash) +{ + class_("Hash") + .def("update", &Hash::update) + .def("crc32", &Hash::crc32) + .def("ed2k", &Hash::ed2k) + .def("md5", &Hash::md5) + .def("sha1", &Hash::sha1); +} diff --git a/pyanidb.conf b/pyanidb.conf new file mode 100644 index 0000000..093e7a4 --- /dev/null +++ b/pyanidb.conf @@ -0,0 +1,5 @@ +# Put this under ~/.pyanidb.conf and chmod 600 to prevent other users from reading your password. + +[auth] +username = foo +password = foossecretpassword \ No newline at end of file -- cgit v1.2.3