#include #include #include #include #include #include #include "parsing.h" #include "channel.h" #include "user.h" #include "word.h" #include "export_xml.h" #include "config.h" #define NICK_BUFFER_SIZE 0x100 #define TEXT_BUFFER_SIZE 0x400 #define LINE_BUFFER_SIZE 0x400 #define TIME_BUFFER_SIZE 0xf #define DATE_BUFFER_SIZE 0x20 static pthread_mutex_t user_mutex, user_time_mutex, word_mutex, channel_mutex, time_mutex; static struct user_t *last_user = NULL; static int in_monolog = 0, monolog_len = 0; static struct tm now_global; static inline void add_word(struct user_t *user, wchar_t *word, int len) { pthread_mutex_lock(&user_mutex); user->words++; pthread_mutex_unlock(&user_mutex); word[len] = '\0'; char mbword[TEXT_BUFFER_SIZE]; wcstombs(mbword, word, TEXT_BUFFER_SIZE); pthread_mutex_lock(&word_mutex); struct word_t *word_s = word_get(mbword); word_s->count++; pthread_mutex_unlock(&word_mutex); } static void process_file(FILE *f, struct channel_t *channel, struct regexset_t *rs) { char line[LINE_BUFFER_SIZE]; const char *log_date_format, *day_date_format; struct tm now; if(rs->log_date_format) { log_date_format = rs->log_date_format; } else if(ircstats_config.log_date_format) { log_date_format = ircstats_config.log_date_format; } else { log_date_format = NULL; } if(rs->day_date_format) { day_date_format = rs->day_date_format; } else if(ircstats_config.day_date_format) { day_date_format = ircstats_config.day_date_format; } else { day_date_format = NULL; } while(fgets(line, LINE_BUFFER_SIZE, f)) { int rc; int ovector[30]; rc = pcre_exec(rs->text, rs->text_e, line, strlen(line), 0, 0, ovector, 30); if(rc > 0) { char nick[NICK_BUFFER_SIZE], text[TEXT_BUFFER_SIZE], hour_s[TIME_BUFFER_SIZE], min_s[TIME_BUFFER_SIZE]; pcre_copy_named_substring(rs->text, line, ovector, rc, "nick", nick, NICK_BUFFER_SIZE); pcre_copy_named_substring(rs->text, line, ovector, rc, "text", text, TEXT_BUFFER_SIZE); pcre_copy_named_substring(rs->text, line, ovector, rc, "hour", hour_s, TIME_BUFFER_SIZE); pcre_copy_named_substring(rs->text, line, ovector, rc, "minute", min_s, TIME_BUFFER_SIZE); pthread_mutex_lock(&user_mutex); struct user_t *user = user_get(nick); if(user == last_user) { monolog_len++; if(!in_monolog && monolog_len >= ircstats_config.monolog_min) { in_monolog = 1; user->monologs++; /* Count first lines. */ user->monolog_lines += monolog_len; } else if(in_monolog) { user->monolog_lines++; } } else { last_user = user; in_monolog = 0; monolog_len = 1; } pthread_mutex_unlock(&user_mutex); /* Calculate array index for lines. */ int hour, min, time_i; hour = atoi(hour_s); min = atoi(min_s); time_i = hour*4 + min / 15; /* Count words. */ wchar_t wtext[TEXT_BUFFER_SIZE]; mbstowcs(wtext, text, TEXT_BUFFER_SIZE); pthread_mutex_lock(&user_mutex); user->characters += wcslen(wtext); user->lines[time_i]++; pthread_mutex_unlock(&user_mutex); pthread_mutex_lock(&time_mutex); now = now_global; pthread_mutex_unlock(&time_mutex); now.tm_hour = hour; now.tm_min = min; time_t now_ut = mktime(&now); pthread_mutex_lock(&user_time_mutex); struct user_time_t *user_time = user_time_get(nick, 0); if(user_time->seen_first == 0) { user_time->seen_first = now_ut; } /* Set if user is not yet seen. */ if(user_time->last_join == 0) { user_time->last_join = now_ut; } /* Add to online total. */ if(user_time->seen_last > 0) { user_time->time_total += now_ut - user_time->seen_last; } user_time->seen_last = now_ut; pthread_mutex_unlock(&user_time_mutex); pthread_mutex_lock(&channel_mutex); channel->hours[time_i]++; pthread_mutex_unlock(&channel_mutex); wchar_t word[TEXT_BUFFER_SIZE]; wchar_t *end = wcschr(wtext, '\0'); *word = '\0'; int len = 0; for(wchar_t *pos = wtext; pos < end; pos++) { if(iswblank(*pos)) { if(len >= ircstats_config.wordlen_min) { add_word(user, word, len); } len = 0; *word = '\0'; } else if(iswalpha(*pos)) { word[len++] = towlower(*pos); } else { len = 0; *word = '\0'; } } if(len >= ircstats_config.wordlen_min) { add_word(user, word, len); } continue; } rc = pcre_exec(rs->join, rs->join_e, line, strlen(line), 0, 0, ovector, 30); if(rc > 0) { char nick[NICK_BUFFER_SIZE], hour_s[TIME_BUFFER_SIZE], min_s[TIME_BUFFER_SIZE]; pcre_copy_named_substring(rs->join, line, ovector, rc, "nick", nick, NICK_BUFFER_SIZE); pcre_copy_named_substring(rs->join, line, ovector, rc, "hour", hour_s, TIME_BUFFER_SIZE); pcre_copy_named_substring(rs->join, line, ovector, rc, "minute", min_s, TIME_BUFFER_SIZE); int hour, min; hour = atoi(hour_s); min = atoi(min_s); pthread_mutex_lock(&time_mutex); now = now_global; pthread_mutex_unlock(&time_mutex); now.tm_hour = hour; now.tm_min = min; time_t now_ut = mktime(&now); pthread_mutex_lock(&user_time_mutex); struct user_time_t *user_time = user_time_get(nick, 0); if(user_time->seen_first == 0) { user_time->seen_first = now_ut; } user_time->last_join = user_time->seen_last = now_ut; pthread_mutex_unlock(&user_time_mutex); continue; } rc = pcre_exec(rs->part, rs->part_e, line, strlen(line), 0, 0, ovector, 30); if(rc > 0) { char nick[NICK_BUFFER_SIZE], hour_s[TIME_BUFFER_SIZE], min_s[TIME_BUFFER_SIZE]; pcre_copy_named_substring(rs->part, line, ovector, rc, "nick", nick, NICK_BUFFER_SIZE); pcre_copy_named_substring(rs->part, line, ovector, rc, "hour", hour_s, TIME_BUFFER_SIZE); pcre_copy_named_substring(rs->part, line, ovector, rc, "minute", min_s, TIME_BUFFER_SIZE); int hour, min; hour = atoi(hour_s); min = atoi(min_s); pthread_mutex_lock(&time_mutex); now = now_global; pthread_mutex_unlock(&time_mutex); now.tm_hour = hour; now.tm_min = min; time_t now_ut = mktime(&now); pthread_mutex_lock(&user_time_mutex); struct user_time_t *user_time = user_time_get(nick, 1); if(user_time->real_user) { struct user_time_t *temp = user_time->real_user; user_time->real_user = NULL; user_time = temp; } if(user_time->seen_first == 0) { user_time->seen_first = now_ut; } /* Add to online total. */ if(user_time->seen_last > 0) { user_time->time_total += now_ut - user_time->seen_last; } user_time->seen_last = now_ut; pthread_mutex_unlock(&user_time_mutex); continue; } rc = pcre_exec(rs->quit, rs->quit_e, line, strlen(line), 0, 0, ovector, 30); if(rc > 0) { char nick[NICK_BUFFER_SIZE], hour_s[TIME_BUFFER_SIZE], min_s[TIME_BUFFER_SIZE]; pcre_copy_named_substring(rs->quit, line, ovector, rc, "nick", nick, NICK_BUFFER_SIZE); pcre_copy_named_substring(rs->quit, line, ovector, rc, "hour", hour_s, TIME_BUFFER_SIZE); pcre_copy_named_substring(rs->quit, line, ovector, rc, "minute", min_s, TIME_BUFFER_SIZE); int hour, min; hour = atoi(hour_s); min = atoi(min_s); pthread_mutex_lock(&time_mutex); now = now_global; pthread_mutex_unlock(&time_mutex); now.tm_hour = hour; now.tm_min = min; time_t now_ut = mktime(&now); pthread_mutex_lock(&user_time_mutex); struct user_time_t *user_time = user_time_get(nick, 1); if(user_time->real_user) { struct user_time_t *temp = user_time->real_user; user_time->real_user = NULL; user_time = temp; } if(user_time->seen_first == 0) { user_time->seen_first = now_ut; } /* Add to online total. */ if(user_time->seen_last > 0) { user_time->time_total += now_ut - user_time->seen_last; } user_time->seen_last = now_ut; pthread_mutex_unlock(&user_time_mutex); continue; } rc = pcre_exec(rs->nick_changed, rs->nick_changed_e, line, strlen(line), 0, 0, ovector, 30); if(rc > 0) { char oldnick[NICK_BUFFER_SIZE], newnick[NICK_BUFFER_SIZE], hour_s[TIME_BUFFER_SIZE], min_s[TIME_BUFFER_SIZE]; pcre_copy_named_substring(rs->nick_changed, line, ovector, rc, "old_nick", oldnick, NICK_BUFFER_SIZE); pcre_copy_named_substring(rs->nick_changed, line, ovector, rc, "new_nick", newnick, NICK_BUFFER_SIZE); pcre_copy_named_substring(rs->nick_changed, line, ovector, rc, "hour", hour_s, TIME_BUFFER_SIZE); pcre_copy_named_substring(rs->nick_changed, line, ovector, rc, "minute", min_s, TIME_BUFFER_SIZE); int hour, min; hour = atoi(hour_s); min = atoi(min_s); pthread_mutex_lock(&time_mutex); now = now_global; pthread_mutex_unlock(&time_mutex); now.tm_hour = hour; now.tm_min = min; time_t now_ut = mktime(&now); pthread_mutex_lock(&user_time_mutex); struct user_time_t *from_user = user_time_get(oldnick, 1); struct user_time_t *to_user = user_time_get(newnick, 1); if(from_user->real_user) { to_user->real_user = from_user->real_user; from_user->real_user = NULL; } else { to_user->real_user = from_user; } if(to_user->real_user) { to_user->time_total += now_ut - to_user->seen_last; to_user->real_user->seen_last = now_ut; } else { from_user->time_total += now_ut - from_user->seen_last; from_user->seen_last = now_ut; } pthread_mutex_unlock(&user_time_mutex); continue; } rc = pcre_exec(rs->log_opened, rs->log_opened_e, line, strlen(line), 0, 0, ovector, 30); if(rc > 0) { char date[DATE_BUFFER_SIZE]; if(!log_date_format) { continue; } pcre_copy_named_substring(rs->log_opened, line, ovector, rc, "date", date, DATE_BUFFER_SIZE); if(!strptime(date, log_date_format, &now)) { printf("log fail: %s\n", date); continue; } pthread_mutex_lock(&time_mutex); now_global = now; pthread_mutex_unlock(&time_mutex); continue; } rc = pcre_exec(rs->day_changed, rs->day_changed_e, line, strlen(line), 0, 0, ovector, 30); if(rc > 0) { char date[DATE_BUFFER_SIZE]; if(!day_date_format) { continue; } pcre_copy_named_substring(rs->day_changed, line, ovector, rc, "date", date, DATE_BUFFER_SIZE); if(!strptime(date, day_date_format, &now)) { printf("day fail: %s\n", date); continue; } pthread_mutex_lock(&time_mutex); now_global = now; pthread_mutex_unlock(&time_mutex); continue; } rc = pcre_exec(rs->kick, rs->kick_e, line, strlen(line), 0, 0, ovector, 30); if(rc > 0) { char nick[NICK_BUFFER_SIZE], victim[NICK_BUFFER_SIZE], hour_s[TIME_BUFFER_SIZE], min_s[TIME_BUFFER_SIZE]; pcre_copy_named_substring(rs->kick, line, ovector, rc, "hour", hour_s, TIME_BUFFER_SIZE); pcre_copy_named_substring(rs->kick, line, ovector, rc, "minute", min_s, TIME_BUFFER_SIZE); pcre_copy_named_substring(rs->kick, line, ovector, rc, "nick", nick, NICK_BUFFER_SIZE); pcre_copy_named_substring(rs->kick, line, ovector, rc, "victim", victim, NICK_BUFFER_SIZE); int hour, min; hour = atoi(hour_s); min = atoi(min_s); pthread_mutex_lock(&time_mutex); now = now_global; pthread_mutex_unlock(&time_mutex); now.tm_hour = hour; now.tm_min = min; time_t now_ut = mktime(&now); pthread_mutex_lock(&user_mutex); struct user_t *user = user_get(nick), *victim_user = user_get(victim); user->kicks++; victim_user->kicked++; pthread_mutex_unlock(&user_mutex); pthread_mutex_lock(&user_time_mutex); struct user_time_t *user_time = user_time_get(nick, 0); if(user_time->seen_first == 0) { user_time->seen_first = now_ut; } /* Add to online total. */ if(user_time->seen_last > 0) { user_time->time_total += now_ut - user_time->seen_last; } user_time->seen_last = now_ut; pthread_mutex_unlock(&user_time_mutex); continue; } } } struct thread_arg_t { FILE *f; struct channel_t *channel; struct regexset_t *rs; }; static void *thread_func(void *arg) { struct thread_arg_t *ta = arg; process_file(ta->f, ta->channel, ta->rs); return NULL; } static void merge_user_time_data(struct user_time_t *user_time) { struct user_t *user; user = user_get(user_time->nick); if(user->seen_first == 0 || user_time->seen_first < user->seen_first) { user->seen_first = user_time->seen_first; } if(user_time->seen_last > user->seen_last) { user->seen_last = user_time->seen_last; } printf("adding %lu to %s\n", user_time->time_total, user->nick); user->time_total += user_time->time_total; } static void merge_time_data() { for(int i = 0; i < USERS_MAX; i++) { struct user_time_t *user_time; user_time = &users_time[i]; if(user_time->nick) merge_user_time_data(user_time); user_time = user_time->next; while(user_time) { merge_user_time_data(user_time); user_time = user_time->next; } } } void process(int thread_n) { pthread_mutex_init(&user_mutex, NULL); pthread_mutex_init(&user_time_mutex, NULL); pthread_mutex_init(&word_mutex, NULL); pthread_mutex_init(&channel_mutex, NULL); pthread_mutex_init(&time_mutex, NULL); /* Parsing stuff goes here. */ for(int chan_i = 0; chan_i < channel_get_count(); chan_i++) { user_init(); user_time_init(); word_init(); struct channel_t *channel = channel_get(chan_i); printf("Channel %s\n", channel->name); struct channel_file_t *file = channel->files; while(file) { struct regexset_t *rs = file->rs; FILE *f = fopen(file->path, "r"); if(!f) { fprintf(stderr, "\tFailed to open %s\n", file->path); file = file->next; continue; } else printf("\tParsing %s\n", file->path); last_user = NULL; in_monolog = monolog_len = 0; pthread_t *threads; threads = malloc(sizeof(pthread_t) * thread_n); struct thread_arg_t ta; ta.f = f; ta.channel = channel; ta.rs = rs; for(int i = 0; i < thread_n; i++) { pthread_create(&threads[i], NULL, thread_func, &ta); } for(int i = 0; i < thread_n; i++) { pthread_join(threads[i], NULL); } free(threads); fclose(f); file = file->next; } merge_time_data(); export_xml(channel, users); user_free(); user_time_free(); word_free(); } pthread_mutex_destroy(&time_mutex); pthread_mutex_destroy(&user_mutex); pthread_mutex_destroy(&user_time_mutex); pthread_mutex_destroy(&word_mutex); pthread_mutex_destroy(&channel_mutex); }