diff --git a/admin.py b/admin.py new file mode 100644 index 0000000..177a410 --- /dev/null +++ b/admin.py @@ -0,0 +1,28 @@ +from django.contrib import admin + +from plugins.reporting.models import * + + +class JournalReportRecipientAdmin(admin.ModelAdmin): + list_display = ('pk', 'user', 'journal',) + list_filter = ('journal',) + raw_id_fields = ('user',) + + +class JournalReportReceiptAdmin(admin.ModelAdmin): + list_display = ('recipient', 'sent_on') + + +class JournalReportStatsAdmin(admin.ModelAdmin): + list_display = ('article', 'date', 'views', 'downloads', 'citations') + list_filter = ('article',) + raw_id_fields = ('article',) + + +admin_list = [ + (JournalReportRecipient, JournalReportRecipientAdmin), + (JournalReportReceipt, JournalReportReceiptAdmin), + (JournalReportStats, JournalReportStatsAdmin), +] + +[admin.site.register(*t) for t in admin_list] \ No newline at end of file diff --git a/logic.py b/logic.py index 8fdb61a..a73760f 100644 --- a/logic.py +++ b/logic.py @@ -1,19 +1,29 @@ import csv import os +import uuid from datetime import date, timedelta from dateutil.relativedelta import relativedelta import datetime from datetime import datetime +from dateutil.rrule import rrule, MONTHLY + +from bs4 import BeautifulSoup from django.utils import timezone from django.conf import settings from django.template.defaultfilters import strip_tags -from django.db.models import Min, Count +from django.db.models import ( + DurationField, + ExpressionWrapper, + F, + Min, + Count, + Avg +) +from django.template.loader import render_to_string from submission import models as sm -from metrics import models as mm -from core.files import serve_temp_file -from core import models as core_models +from core.files import serve_temp_file, get_temp_file_path_from_name from utils.function_cache import cache from journal import models as jm from review import models as rm @@ -59,8 +69,8 @@ def get_start_and_end_months(request): 'end_month', get_current_month_year() ) - start_month_m, start_month_y = start_month.split('-') - end_month_m, end_month_y = end_month.split('-') + start_month_y, start_month_m = start_month.split('-') + end_month_y, end_month_m = end_month.split('-') date_parts = { 'start_month_m': start_month_m, @@ -104,6 +114,37 @@ def get_articles(journal, start_date, end_date): return articles +def get_articles_with_counts(journal, start_date, end_date): + if journal: + articles = sm.Article.objects.filter( + date_published__lte=end_date, + journal=journal + ).select_related('section') + else: + articles = sm.Article.objects.filter( + date_published__lte=end_date, + ).select_related('section') + + for article in articles: + article.views = mm.ArticleAccess.objects.filter( + article=article, + accessed__gte=start_date, + accessed__lte=end_date, + type='view' + ).count() + article.downloads = mm.ArticleAccess.objects.filter( + article=article, + accessed__gte=start_date, + accessed__lte=end_date, + type='download' + ).count() + article.citations = mm.ArticleLink.objects.filter( + article=article + ).count() + + return articles + + def get_accesses(journal, start_date, end_date): views = mm.ArticleAccess.objects.filter( article__journal=journal, @@ -335,11 +376,11 @@ def export_country_csv(metrics): @cache(300) -def get_most_viewed_article(metrics): +def get_most_viewed_article(metrics, count=1): from django.db.models import Count return metrics.values('article__title').annotate( - total=Count('article')).order_by('-total')[:1] + total=Count('article')).order_by('-total')[:count] @cache(300) @@ -411,7 +452,7 @@ def export_press_csv(data_dict): for data in data_dict: most_viewed_article_string = '{title} ({count})'.format( - title=data['most_viewed_article'][0]['article__title'] if data['most_viewed_article'] else '', + title=data['most_viewed_article'][0]['article__title'] if data['most_viewed_article'] else '', count=data['most_viewed_article'][0]['total'] if data['most_viewed_article'] else '' ) @@ -595,3 +636,247 @@ def get_journal_citations(journal): journal.citation_count = counter return articles + + +@cache(600) +def get_book_data(date_parts): + """ + Checks if the books plugin logic module can be loaded or returns an empty dict. + """ + + try: + from plugins.books import models as book_models, logic as book_logic + books = book_models.Book.objects.all() + return book_logic.book_metrics_by_month(books, date_parts) + except ImportError as e: + print(e) + return [], [], '', '' + + +def get_months_between_date_parts(date_parts): + start_dt = datetime(year=int(date_parts.get('start_month_y')), month=int(date_parts.get('start_month_m')), day=1) + end_dt = datetime(year=int(date_parts.get('end_month_y')), month=int(date_parts.get('end_month_m')), day=1) + + return [dt for dt in rrule(MONTHLY, dtstart=start_dt, until=end_dt)] + + +def get_months_to_jan(date_parts): + start_dt = datetime(int(date_parts.get('end_month_y')), 1, 1) + end_dt = datetime(int(date_parts.get('end_month_y')), int(date_parts.get('end_month_m')), 1) + + return [dt for dt in rrule(MONTHLY, dtstart=start_dt, until=end_dt)] + + +@cache(600) +def get_average_review_time(journal, start, end): + f_review_delta = ExpressionWrapper( + F('date_complete') - F('date_requested'), + output_field=DurationField(), + ) + average_time_to_complete = rm.ReviewAssignment.objects.filter( + article__journal=journal, + date_requested__gte=start, + date_requested__lte=end, + date_complete__gte=start, + date_complete__lte=end, + ).annotate( + editorial_delta=f_review_delta + ).aggregate( + Avg('editorial_delta') + ) + + return average_time_to_complete.get('editorial_delta__avg', 0) + + +@cache(600) +def get_board_report_journal_date(date_parts, all_metrics): + journals = jm.Journal.objects.all() + start_str = '{}-01'.format(date_parts.get('start_unsplit')) + end_str = '{}-27'.format(date_parts.get('end_unsplit')) + + start = datetime.strptime(start_str, '%Y-%m-%d') + end = datetime.strptime(end_str, '%Y-%m-%d') + + start = timezone.make_aware(start) + end = timezone.make_aware(end) + + current_year = date_parts.get('end_month_y') + previous_year = str(int(current_year) - 1) + + months_between_dates = len(get_months_between_date_parts(date_parts)) + months_to_jan = len(get_months_to_jan(date_parts)) + + data = [] + for journal in journals: + journal_metrics = all_metrics.filter( + article__journal=journal, + ) + articles = sm.Article.objects.filter( + journal=journal, + ) + submissions = articles.exclude( + stage=sm.STAGE_PUBLISHED, + ).filter( + date_submitted__gte=start, + date_submitted__lte=end, + ) + published_articles = articles.filter( + stage=sm.STAGE_PUBLISHED, + date_published__gte=start, + date_published__lte=end, + ) + authors = sm.FrozenAuthor.objects.filter( + article__in=articles, + ).count() + review_average = get_average_review_time( + journal, + start, + end, + ) + + journal_data = { + 'journal': journal, + 'views': {}, + 'downloads': {}, + 'accesses': {}, + 'total_accesses': journal_metrics.count(), + 'articles': {} + } + + journal_data['views']['total'] = journal_metrics.filter(type='view').count() + journal_data['views']['period'] = journal_metrics.filter( + type='view', + article__stage=sm.STAGE_PUBLISHED, + accessed__year__gte=date_parts.get('start_month_y'), + accessed__month__gte=date_parts.get('start_month_m'), + accessed__year__lte=date_parts.get('end_month_y'), + accessed__month__lte=date_parts.get('end_month_m'), + ).count() + journal_data['views']['period_average'] = round(journal_data['views']['period'] / months_between_dates, 2) + journal_data['views']['year'] = journal_metrics.filter( + type='view', + article__stage=sm.STAGE_PUBLISHED, + accessed__year=current_year, + ).count() + journal_data['views']['year_average'] = round(journal_data['views']['year'] / months_to_jan, 2) + + journal_data['downloads']['total'] = journal_metrics.filter(type='download').count() + journal_data['downloads']['period'] = journal_metrics.filter( + type='download', + article__stage=sm.STAGE_PUBLISHED, + accessed__year__gte=date_parts.get('start_month_y'), + accessed__month__gte=date_parts.get('start_month_m'), + accessed__year__lte=date_parts.get('end_month_y'), + accessed__month__lte=date_parts.get('end_month_m'), + ).count() + journal_data['downloads']['period_average'] = round(journal_data['downloads']['period'] / months_between_dates, 2) + journal_data['downloads']['year'] = journal_metrics.filter( + type='download', + article__stage=sm.STAGE_PUBLISHED, + accessed__year=current_year, + ).count() + journal_data['downloads']['year_average'] = round(journal_data['downloads']['year'] / months_to_jan, 2) + + journal_data['accesses']['total'] = journal_data['views']['total'] + journal_data['downloads']['total'] + journal_data['accesses']['period'] = journal_data['views']['period'] + journal_data['downloads']['period'] + journal_data['accesses']['period_average'] = round(journal_data['accesses']['total'] / months_between_dates, 2) + journal_data['accesses']['year'] = journal_data['views']['year'] + journal_data['downloads']['year'] + + journal_data['articles']['all'] = articles + journal_data['articles']['submissions'] = submissions.count() + journal_data['articles']['publications'] = published_articles.count() + journal_data['articles']['authors'] = authors + journal_data['review_average'] = review_average.days if review_average else 'N/a' + + data.append(journal_data) + + return data + + +@cache(600) +def most_accessed_articles(date_parts, all_metrics): + top_viewed_articles = all_metrics.filter( + accessed__year__gte=date_parts.get('start_month_y'), + accessed__month__gte=date_parts.get('start_month_m'), + accessed__year__lte=date_parts.get('end_month_y'), + accessed__month__lte=date_parts.get('end_month_m'), + ).values('article').annotate( + total=Count('article')).order_by('-total')[:10] + + articles = [] + for article in top_viewed_articles: + article = sm.Article.objects.get( + pk=article.get('article'), + ) + article_metrics = all_metrics.filter( + article=article, + accessed__year__gte=date_parts.get('start_month_y'), + accessed__month__gte=date_parts.get('start_month_m'), + accessed__year__lte=date_parts.get('end_month_y'), + accessed__month__lte=date_parts.get('end_month_m'), + ) + article.accesses = article_metrics.count() + article.views = article_metrics.filter(type='view').count() + article.downloads = article_metrics.filter(type='download').count() + article.citations = mm.ArticleLink.objects.filter(article=article).count() + articles.append(article) + + return articles + + +def html_table_to_csv(html): + filename = '{0}.csv'.format(uuid.uuid4()) + filepath = get_temp_file_path_from_name( + filename, + ) + soup = BeautifulSoup(str(html), 'lxml') + with open(filepath, "w", encoding="utf-8") as f: + wr = csv.writer(f) + for table in soup.find_all("table"): + for row in table.find_all("tr"): + cells = [cell.string for cell in row.findChildren(['th', 'td'])] + wr.writerow(cells) + wr.writerow([]) + + f.close() + return filepath, filename + + +def export_board_report_csv( + request, + book_data, + journal_data, + most_accessed_articles, + start_month, + end_month, + book_dates, + current_year, + previous_year, +): + elements = [ + 'header.html', + 'books.html', + 'journals.html', + 'articles.html', + ] + + context = { + 'request': request, + 'start_month': start_month, + 'end_month': end_month, + 'book_data': book_data, + 'journal_data': journal_data, + 'most_accessed_articles': most_accessed_articles, + 'book_dates': book_dates, + 'current_year': current_year, + 'previous_year': previous_year, + } + + html = '' + for element in elements: + html = html + render_to_string( + 'reporting/elements/{element}'.format(element=element), + context, + ) + csv_filepath, csv_filename = html_table_to_csv(html) + return serve_temp_file(csv_filepath, csv_filename) diff --git a/management/__init__.py b/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/management/commands/__init__.py b/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/management/commands/generate_journal_report_data.py b/management/commands/generate_journal_report_data.py new file mode 100644 index 0000000..e69de29 diff --git a/management/commands/send_monthly_journal_report.py b/management/commands/send_monthly_journal_report.py new file mode 100644 index 0000000..a85127b --- /dev/null +++ b/management/commands/send_monthly_journal_report.py @@ -0,0 +1,104 @@ +import csv +from dateutil.relativedelta import relativedelta + +from django.core.management.base import BaseCommand +from django.utils import timezone, translation +from django.conf import settings +from django.core.mail import EmailMultiAlternatives +from django.utils.html import strip_tags + +from plugins.reporting import models, logic +from submission import models as submission_models +from journal import models as journal_models +from core import files +from utils import setting_handler + +HEADERS = [ + 'Article Title', + 'Section', + 'Date Submitted', + 'Date Accepted', + 'Date Published', + 'Views', + 'Downloads', + 'Citations', + 'Total Accesses', +] + + +def get_dates(): + today = timezone.now() + d = today - relativedelta(months=1) + + first_day = timezone.datetime(d.year, d.month, 1) + last_day = timezone.datetime(today.year, today.month, 1) - relativedelta(days=1) + return timezone.make_aware(first_day), timezone.make_aware(last_day) + + +def send_email(recipient, journal, csv_path): + from_email = setting_handler.get_setting('general', 'from_address', journal).value + from_string = "{} <{}>".format(journal.name, from_email) + subject = 'Journal Monthly Report' + html = "

Please find the monthly report for {} attached.

".format(journal.name) + + msg = EmailMultiAlternatives(subject, strip_tags(html), from_string, to=[recipient.user.email]) + msg.attach_alternative(html, "text/html") + + with open(csv_path) as file: + msg.attach(file.name, file.read(), 'text/csv') + + return msg.send() + + +class Command(BaseCommand): + """ Sends out a monthly journal report to recipients.""" + + help = "Sends out a monthly journal report to recipients" + + def add_arguments(self, parser): + parser.add_argument('--journal_code') + + def handle(self, *args, **options): + translation.activate(settings.LANGUAGE_CODE) + journal_code = options.get('journal_code') + journals = journal_models.Journal.objects.all() + start_date, end_date = get_dates() + + if journal_code: + journals = journals.filter(code=journal_code) + + if not journals: + print('No journals found.') + exit() + + for journal in journals: + recipients = models.JournalReportRecipient.objects.filter(journal=journal) + + # Generate a CSV for each journal + csv_file_path = files.get_temp_file_path_from_name('journal_{}.csv'.format(journal.code)) + with open(csv_file_path, "w") as f: + wr = csv.writer(f, quoting=csv.QUOTE_ALL) + wr.writerow(HEADERS) + + for article in logic.get_articles_with_counts(journal, start_date, end_date): + row = [ + article.title, + article.section.name if article.section else '', + article.date_submitted, + article.date_accepted, + article.date_published, + article.views, + article.downloads, + article.citations, + article.views + article.downloads, + ] + wr.writerow(row) + + if not recipients: + print('No recipients found for {}'.format(journal.name)) + continue + + for recipient in recipients: + send_email(recipient, journal, csv_file_path) + + files.unlink_temp_file(csv_file_path) diff --git a/migrations/0001_initial.py b/migrations/0001_initial.py new file mode 100644 index 0000000..56fea49 --- /dev/null +++ b/migrations/0001_initial.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2021-06-07 11:59 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('journal', '0041_issue_short_description'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('submission', '0052_auto_20210525_1743'), + ] + + operations = [ + migrations.CreateModel( + name='JournalReportReceipt', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('sent_on', models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.CreateModel( + name='JournalReportRecipient', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('journal', models.ForeignKey(help_text='The journal that the user will receive the report for', on_delete=django.db.models.deletion.CASCADE, to='journal.Journal')), + ('user', models.ForeignKey(help_text='The user who will receive the report', on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='JournalReportStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date', models.DateField()), + ('views', models.PositiveIntegerField()), + ('downloads', models.PositiveIntegerField()), + ('citations', models.PositiveIntegerField()), + ('article', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='submission.Article')), + ], + ), + migrations.AddField( + model_name='journalreportreceipt', + name='recipient', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='reporting.JournalReportRecipient'), + ), + ] diff --git a/migrations/__init__.py b/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models.py b/models.py new file mode 100644 index 0000000..1e7e9ce --- /dev/null +++ b/models.py @@ -0,0 +1,25 @@ +from django.db import models + + +class JournalReportRecipient(models.Model): + user = models.ForeignKey( + 'core.Account', + help_text="The user who will receive the report", + ) + journal = models.ForeignKey( + 'journal.Journal', + help_text="The journal that the user will receive the report for", + ) + + +class JournalReportReceipt(models.Model): + recipient = models.ForeignKey(JournalReportRecipient) + sent_on = models.DateTimeField(auto_now_add=True) + + +class JournalReportStats(models.Model): + article = models.ForeignKey('submission.Article') + date = models.DateField() + views = models.PositiveIntegerField() + downloads = models.PositiveIntegerField() + citations = models.PositiveIntegerField() diff --git a/templates/reporting/elements/articles.html b/templates/reporting/elements/articles.html new file mode 100644 index 0000000..beaa602 --- /dev/null +++ b/templates/reporting/elements/articles.html @@ -0,0 +1,20 @@ + + + + + + + + + + {% for article in most_accessed_articles %} + + + + + + + + + {% endfor %} +
TitleJournal NameAccessesViewsDownloadsCitations (all time)
{{ article.title }}{{ article.journal.name }}{{ article.accesses }}{{ article.views }}{{ article.downloads }}{{ article.citations }}
\ No newline at end of file diff --git a/templates/reporting/elements/books.html b/templates/reporting/elements/books.html new file mode 100644 index 0000000..3bb47cb --- /dev/null +++ b/templates/reporting/elements/books.html @@ -0,0 +1,30 @@ +{% load dict %} + + + + + + {% for date in book_dates %} + + {% endfor %} + + + + + + {% for d in book_data %} + + + {% for dm in d.date_metrics %} + + {% endfor %} + + + + {% endfor %} + +
Book{{ date.month }} {{ date.year }}{{ current_year }} Total{{ previous_year }} Total
{{ d.book.title }}{{ dm }} + {% tag_get d current_year %} + + {% tag_get d previous_year %} +
\ No newline at end of file diff --git a/templates/reporting/elements/header.html b/templates/reporting/elements/header.html new file mode 100644 index 0000000..30539b1 --- /dev/null +++ b/templates/reporting/elements/header.html @@ -0,0 +1,8 @@ + + + + + + + +
Press Board Report for {{ request.press }}. {{ start_month }}-{{ end_month }}
\ No newline at end of file diff --git a/templates/reporting/elements/journals.html b/templates/reporting/elements/journals.html new file mode 100644 index 0000000..c243334 --- /dev/null +++ b/templates/reporting/elements/journals.html @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {% for journal in journal_data %} + + + + + + + + + + + + + + + + + + + + + + + {% endfor %} +
ViewsDownloadsTotal Accesses + Article Data
Journal NameAll TimeYearIn PeriodPeriod AvgAll TimeYearIn PeriodPeriod AvgAll TimeYearIn PeriodPeriod AvgSubmissionsPublicationAuthorsPeer Review Turnaround Avg (days)
{{ journal.journal.name }}{{ journal.views.total }}{{ journal.views.year }}{{ journal.views.period }}{{ journal.views.period_average }}{{ journal.downloads.total }}{{ journal.downloads.year }}{{ journal.downloads.period }}{{ journal.downloads.period_average }}{{ journal.accesses.total }}{{ journal.accesses.year }}{{ journal.accesses.period }}{{ journal.accesses.period_average }}{{ journal.articles.submissions }}{{ journal.articles.publications }}{{ journal.articles.authors }}{{ journal.review_average }}
\ No newline at end of file diff --git a/templates/reporting/index.html b/templates/reporting/index.html index d85ecc2..f04f2d6 100644 --- a/templates/reporting/index.html +++ b/templates/reporting/index.html @@ -23,6 +23,13 @@ View Report +
  • + Press Board Report +
    +

    Shows information at the press level useful for reporting to editorial boards.

    + View Report +
    +
  • Journal Usage by Month Report
    diff --git a/templates/reporting/press_board_report.html b/templates/reporting/press_board_report.html new file mode 100644 index 0000000..350590d --- /dev/null +++ b/templates/reporting/press_board_report.html @@ -0,0 +1,72 @@ +{% extends "admin/core/base.html" %} +{% load dict %} + +{% block title %}Reports{% endblock %} +{% block title-section %}Press Board Report{% endblock %} + +{% block breadcrumbs %} + {{ block.super }} +
  • Reporting Index
  • +
  • Press Board Report
  • +{% endblock %} + +{% block body %} +
    +
    +
    +
    +

    Date Filters

    +
    + {% csrf_token %} + +
    +
    +
    +
    + {{ month_form.errors|safe }} +
    + {{ month_form.start_month }} +
    +
    + {{ month_form.end_month }} +
    +
    + +
    +
    +


    +
    +
    +
    +
    + {% if book_data and book_dates %} +
    +
    +

    Press Board Report

    +
    +
    + {% include "reporting/elements/books.html" %} +
    +
    + {% endif %} +
    +
    +

    Journals

    +
    +
    + {% include "reporting/elements/journals.html" %} +
    +
    +
    +
    +

    Most Accessed Articles

    +
    +
    + {% include "reporting/elements/articles.html" %} +
    +
    +
    + +
    + +{% endblock %} diff --git a/urls.py b/urls.py index c206b8e..ac73115 100644 --- a/urls.py +++ b/urls.py @@ -9,6 +9,9 @@ url(r'^press/$', views.press, name='reporting_press'), + url(r'^press/board_report/$', + views.press_board_report, + name='reporting_press_board_report'), url(r'^by_month/$', views.report_journal_usage_by_month, name='reporting_journal_usage_by_month'), diff --git a/views.py b/views.py index 8afd75f..c51f7e2 100644 --- a/views.py +++ b/views.py @@ -1,5 +1,5 @@ from django.shortcuts import render, reverse, redirect, get_object_or_404 -from django.db.models import Q +from django.contrib.admin.views.decorators import staff_member_required from plugins.reporting import forms, logic from journal import models @@ -350,3 +350,45 @@ def report_article_citing_works(request, journal_id, article_id): return render(request, template, context) + +@staff_member_required +def press_board_report(request): + """ + Exports a lot of information useful for making reports to editorial boards. Slow. + """ + start_month, end_month, date_parts = logic.get_start_and_end_months(request) + month_form = forms.MonthForm( + initial={ + 'start_month': start_month, 'end_month': end_month, + } + ) + all_metrics = mm.ArticleAccess.objects.all() + book_data, book_dates, current_year, previous_year = logic.get_book_data(date_parts) + journal_data = logic.get_board_report_journal_date(date_parts, all_metrics) + most_accessed_articles = logic.most_accessed_articles(date_parts, all_metrics) + + if request.POST: + return logic.export_board_report_csv( + request, + book_data, + journal_data, + most_accessed_articles, + start_month, + end_month, + book_dates, + current_year, + previous_year, + ) + + template = 'reporting/press_board_report.html' + context = { + 'month_form': month_form, + 'book_data': book_data, + 'book_dates': book_dates, + 'current_year': current_year, + 'previous_year': previous_year, + 'journal_data': journal_data, + 'most_accessed_articles': most_accessed_articles, + } + return render(request, template, context) +