# -*- coding: utf-8 -*-
# Akvo RSR is covered by the GNU Affero General Public License.
# See more details in the license.txt file located at the root folder of the Akvo RSR module.
# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.
import os
import django
os.environ['DJANGO_SETTINGS_MODULE'] = 'akvo.settings'
from akvo import settings
from akvo.scripts.cordaid import ERROR_EXCEPTION, ERROR_CREATE_ORG, ERROR_UPLOAD_ORG, ACTION_CREATE_ORG, log, init_log, print_log, ACTION_CREATE_IOI, ACTION_UPDATE_ORG
import getopt
import json
import sys
from lxml import etree
from rest_framework.status import HTTP_200_OK, HTTP_201_CREATED
from akvo.scripts.cordaid.requester import Requester
API_VERSION = 'v1'
# get this module
me = sys.modules[__name__]
api_settings = dict(
UPLOAD_ROOT_DIR='/Users/gabriel/Downloads/cordaid_full',
PROJECT_IMAGES_SUBDIR='project_images',
LOGOS_SUBDIR='logos',
IATI_ACTIVITES_FILENAME='iati_export0.xml',
ORGANISATIONS_FILENAME='cordaid_organisations.xml',
ORGANISATIONS_UPLOAD_LOG_FILENAME='organisations_upload_{datetime}.csv',
)
# construct local variables for Cordaid supporting data
for key, val in api_settings.items():
# try to grab the identifier from settings, if not found use the default from cordaid_settings
setattr(me, key, getattr(settings, key, val))
IATI_ACTIVITIES_XML = os.path.join(me.UPLOAD_ROOT_DIR, me.IATI_ACTIVITES_FILENAME)
ORGANISATIONS_XML = os.path.join(me.UPLOAD_ROOT_DIR, me.ORGANISATIONS_FILENAME)
ORGANISATIONS_UPLOAD_LOG_FILE = os.path.join(
me.UPLOAD_ROOT_DIR, me.ORGANISATIONS_UPLOAD_LOG_FILENAME
)
[docs]def user_org(user_cred):
try:
profile = Requester(
url_template="http://{domain}/api/{api_version}/user/?"
"format=json&api_key={api_key}&username={username}&"
"user__username={username}",
url_args=user_cred
)
# find the organisation ID in the path string, e.g. "/api/v1/organisation/42/"
# non-intuitively split() returns an empty string first and last, thus [-2]
return profile.response.json()['objects'][0]['organisation'].split('/')[-2]
except Exception as e:
print("{message}".format(message=e))
return False, None
[docs]def find_org(user_cred, reporting_org_id, internal_org_id):
"""
"""
url_args = user_cred
url_args.update(
recording_org=reporting_org_id,
identifier=internal_org_id,
)
try:
ioi = Requester(
url_template="http://{domain}/rest/v1/internal_organisation_id/?"
"recording_org={recording_org}&identifier={identifier}&format=json",
url_args=url_args,
headers={
'content-type': 'application/xml', 'encoding': 'utf-8',
'Authorization': 'Token {}'.format(user_cred['api_key'])
},
)
# TODO: check that we only get one object back
org_id = ioi.response.json()[0]['referenced_org']
except Exception as e:
print("{message}".format(message=e))
return False, None
return True, org_id
[docs]def post_an_org(org_element, user_cred):
internal_org_id = org_element.find('org_id').text
try:
organisation = Requester(
method='post',
url_template="http://{domain}/rest/v1/organisation/",
url_args=user_cred,
headers={'content-type': 'application/xml', 'encoding': 'utf-8', 'Authorization': 'Token {}'.format(user_cred['api_key'])},
data=etree.tostring(org_element),
accept_codes=[HTTP_201_CREATED]
)
except Exception as e:
return False, "{extra}", dict(
internal_org_id=internal_org_id,
event=ERROR_EXCEPTION,
extra=str(e),
)
if organisation.response.status_code is HTTP_201_CREATED:
import pdb
pdb.set_trace()
return True, "Created organisation ID: {pk}", dict(
pk=organisation.response.json()['id'], event=ACTION_CREATE_ORG
)
elif organisation.response.status_code != HTTP_201_CREATED:
import pdb
pdb.set_trace()
return False, "**** Error creating organisation: {internal_org_id}", dict(
internal_org_id=internal_org_id,
event=ERROR_CREATE_ORG,
extra=organisation.response.text
)
else:
return (
False,
"**** Error creating organisation: {iati_org_id}. HTTP status code: {extra}", dict(
internal_org_id=internal_org_id,
event=ERROR_UPLOAD_ORG,
extra=organisation.response.status_code,
)
)
[docs]def post_an_internal_id(user_cred, reporting_org_id, internal_identifier, pk):
try:
internal_org_id = Requester(
method='post',
url_template="http://{domain}/rest/v1/internal_organisation_id/",
url_args=user_cred,
headers={'content-type': 'application/json', 'encoding': 'utf-8', 'Authorization': 'Token {}'.format(user_cred['api_key'])},
data=json.dumps(dict(
recording_org=reporting_org_id,
referenced_org=pk,
identifier=internal_identifier,
)),
accept_codes=[HTTP_201_CREATED]
)
except Exception as e:
return False, "{extra}", dict(
pk,
event=ERROR_EXCEPTION,
extra=str(e),
)
if internal_org_id.response.status_code is HTTP_201_CREATED:
import pdb
pdb.set_trace()
return True, "Created internal organisation ID: {identifier}", dict(
pk=internal_org_id.response.json()['identifier'],
event=ACTION_CREATE_IOI
)
[docs]def put_an_org(org_element, user_cred, pk):
internal_org_id = org_element.find('org_id').text
user_cred.update(pk=pk)
import pdb
pdb.set_trace()
try:
organisation = Requester(
method='put',
url_template="http://{domain}/rest/v1/organisation/{pk}/",
url_args=user_cred,
headers={'content-type': 'application/xml', 'encoding': 'utf-8', 'Authorization': ' Token {}'.format(user_cred['api_key'])},
data=etree.tostring(org_element),
accept_codes=[HTTP_200_OK]
)
except Exception as e:
return False, "{extra}", dict(
internal_org_id=internal_org_id,
event=ERROR_EXCEPTION,
extra=str(e),
)
if organisation.response.status_code is HTTP_200_OK:
return True, "Updated organisation ID: {pk}", dict(
pk=organisation.response.json()['id'], event=ACTION_UPDATE_ORG,
)
else:
return (
False,
"**** Error updating organisation: {pk}. HTTP status code: {extra}", dict(
pk=pk,
event=ERROR_UPLOAD_ORG,
extra=organisation.response.status_code,
)
)
[docs]def usage(script_name):
print(
"\nUsage: %s <domain> <username> [options]\n\n"
" <domain> The domain you are posting to, e.g. test.akvo.org\n"
" <username> Your Akvo account username\n\n"
" Options (note that either PWD or KEY must be supplied):\n"
" -h, --help show this message\n\n"
" -p PWD, --password=PWD\n"
" Supply your Akvo account password\n\n"
" -k KEY, --api_key=KEY\n"
" Supply the API key generated in your Akvo user profile\n"
% script_name)
[docs]def api_user(domain, username, password='', api_key=''):
user = dict(domain=domain, username=username, api_version=API_VERSION,)
if api_key:
user['api_key'] = api_key
return user
elif password:
auth = Requester(
method='post',
url_template="http://{domain}/auth/token/",
url_args=dict(domain=domain),
data=dict(username=username, password=password),
)
xml = auth.response.text
root = etree.fromstring(xml)
user['api_key'] = root.find("api_key").text
return user
else:
raise Exception("Either password or API key must be supplied")
[docs]def credentials_from_args(argv):
try:
opts, args = getopt.getopt(argv[1:], "hp:k:", ["help", "password=", "api_key="])
except getopt.GetoptError as e:
print(str(e))
usage(argv[0])
sys.exit(2)
kwargs = {}
for opt, arg in opts:
if opt in ("-h", "--help"):
usage(argv[0])
sys.exit()
# TODO: see if it's possible to suppress password echoing in terminal
elif opt in ("-p", "--password"):
kwargs['password'] = arg
elif opt in ("-k", "--api_key"):
kwargs['api_key'] = arg
try:
domain = args[0]
username = args[1]
except IndexError:
usage(argv[0])
sys.exit(2)
try:
user = api_user(domain, username, **kwargs)
return user
except Exception as e:
print("{message}".format(message=e))
usage(argv[0])
return None
[docs]def upload_organisations(argv):
user_cred = credentials_from_args(argv)
reporting_org_id = user_org(user_cred, **dict())
if user_cred:
with open(ORGANISATIONS_XML, 'r') as f:
root = etree.fromstring(f.read())
organisations = root.findall('object')
for i in range(len(organisations)):
internal_org_id = organisations[i].find('org_id').text
name = organisations[i].find('name').text
print("Processing organisation {name}".format(name=name), end=' ')
if internal_org_id:
print(" ID: {org_id}".format(org_id=internal_org_id), end=' ')
import pdb
pdb.set_trace()
ok, pk = find_org(user_cred, reporting_org_id, internal_org_id)
if pk:
ok, message, data = put_an_org(organisations[i], user_cred, pk)
log(message, data)
print(message.format(**data))
else:
ok, message, data = post_an_org(organisations[i], user_cred)
log(message, data)
print(message.format(**data))
if ok:
post_an_internal_id(user_cred, reporting_org_id, internal_org_id, data['pk'])
else:
print("No internal org ID")
if __name__ == '__main__':
django.setup()
import pdb
pdb.set_trace()
upload_organisations(sys.argv)
log_file = init_log(ORGANISATIONS_UPLOAD_LOG_FILE)
names = ('pk', 'other_id', 'event', 'extra')
print_log(log_file, names)
STATIC_ROOT = "/var/akvo/rsr/static/"
STATIC_URL = "/static/"