pagekite: Move to using python3-augeas

- Merge actions/pagekite_util.py into plinth/modules/pagekite/util.py.

- Rename plinth/modules/pagekite/util.py to utils.py.

- Add python-augeus as dependency.

- Move actions/util.py to plinth/action_utils.py and update services
  that use it.

- Rename _run() method to run() as it is being used publicly.

- Import the utils in a more conventional manner.

- Move all python2 bits to python3.
This commit is contained in:
Sunil Mohan Adapa 2015-07-09 23:16:04 +05:30 committed by James Valleroy
parent 2b86ce9fdb
commit 517c364559
14 changed files with 157 additions and 191 deletions

View File

@ -110,6 +110,8 @@ infrastructure in place for it from the start. Use it like this:
## Dependencies
* *Augeas* - Round trip configuration editing utilities
* *Bootstrap Form* - Render Django forms for Twitter Bootstrap
* *CherryPy3* - WSGI web server since Django does not have proper web server

View File

@ -5,10 +5,10 @@
On a Debian based system, run:
$ sudo apt-get install libjs-jquery libjs-modernizr libjs-bootstrap \
make pandoc python3 python3-bootstrapform python3-cherrypy3 \
python3-coverage python3-django python3-django-stronghold python3-gi \
python3-setuptools python3-yaml gir1.2-glib-2.0 gir1.2-networkmanager-1.0 \
gir1.2-packagekitglib-1.0
make pandoc python3 python3-augeas python3-bootstrapform \
python3-cherrypy3 python3-coverage python3-django \
python3-django-stronghold python3-gi python3-setuptools python3-yaml \
gir1.2-glib-2.0 gir1.2-networkmanager-1.0 gir1.2-packagekitglib-1.0
2. Install Plinth:

View File

@ -1,4 +1,4 @@
#!/usr/bin/python2
#!/usr/bin/python3
# -*- mode: python -*-
#
# This file is part of Plinth.
@ -18,9 +18,7 @@
#
"""
Configuration helper for Plinth PageKite interface
Unfortunately there is no python3 package for augeas yet
Configuration helper for Plinth PageKite interface.
"""
import argparse
@ -29,20 +27,19 @@ import json
import os
import subprocess
import util
from pagekite_util import SERVICE_PARAMS, convert_service_to_string, \
get_augeas_servicefile_path, load_service, CONF_PATH
from plinth import action_utils
from plinth.modules.pagekite import utils
aug = augeas.Augeas()
PATHS = {
'service_on': os.path.join(CONF_PATH, '*', 'service_on', '*'),
'kitename': os.path.join(CONF_PATH, '10_account.rc', 'kitename'),
'kitesecret': os.path.join(CONF_PATH, '10_account.rc', 'kitesecret'),
'abort_not_configured': os.path.join(CONF_PATH, '10_account.rc',
'service_on': os.path.join(utils.CONF_PATH, '*', 'service_on', '*'),
'kitename': os.path.join(utils.CONF_PATH, '10_account.rc', 'kitename'),
'kitesecret': os.path.join(utils.CONF_PATH, '10_account.rc', 'kitesecret'),
'abort_not_configured': os.path.join(utils.CONF_PATH, '10_account.rc',
'abort_not_configured'),
'defaults': os.path.join(CONF_PATH, '20_frontends.rc', 'defaults'),
'frontend': os.path.join(CONF_PATH, '20_frontends.rc', 'frontend'),
'defaults': os.path.join(utils.CONF_PATH, '20_frontends.rc', 'defaults'),
'frontend': os.path.join(utils.CONF_PATH, '20_frontends.rc', 'frontend'),
}
@ -93,13 +90,13 @@ def _service(action):
def subcommand_is_running(_):
"""Print whether pagekite is enabled (yes or no)"""
print 'yes' if util.service_is_running('pagekite') else 'no'
print('yes' if action_utils.service_is_running('pagekite') else 'no')
def subcommand_restart(_):
"""Restart the pagekite service"""
_service('restart')
print 'restarted'
print('restarted')
def subcommand_start_and_enable(_):
@ -107,23 +104,23 @@ def subcommand_start_and_enable(_):
aug.save()
# 'start' alone sometimes fails, even if the service is not running
_service('restart')
print 'enabled'
print('enabled')
def subcommand_stop_and_disable(_):
_service('stop')
aug.set(PATHS['abort_not_configured'], '')
aug.save()
print 'disabled'
print('disabled')
def subcommand_get_frontend(_):
"""Get pagekite frontend url"""
if aug.match(PATHS['defaults']):
print "pagekite.net"
print("pagekite.net")
else:
url = aug.get(PATHS['frontend'])
print url if url else ""
print(url or '')
def subcommand_set_frontend(arguments):
@ -144,20 +141,20 @@ def enable_pagekitenet_frontend():
aug.set(PATHS['defaults'], '')
aug.remove(PATHS['frontend'])
aug.save()
print "enabled"
print("enabled")
def subcommand_get_services(arguments):
""" lists all available (enabled) services """
for match in aug.match(PATHS['service_on']):
service = dict([(param, aug.get(os.path.join(match, param)))
for param in SERVICE_PARAMS])
print json.dumps(service)
for param in utils.SERVICE_PARAMS])
print(json.dumps(service))
def subcommand_remove_service(arguments):
"""Searches and removes the service(s) that match all given parameters"""
service = load_service(arguments.service)
service = utils.load_service(arguments.service)
paths = get_existing_service_paths(service)
# TODO: theoretically, everything to do here is:
# [aug.remove(path) for path in paths]
@ -193,7 +190,7 @@ def get_existing_service_paths(service):
def subcommand_add_service(arguments):
"""Add one service"""
service = load_service(arguments.service)
service = utils.load_service(arguments.service)
if get_existing_service_paths(service):
msg = "Service with the parameters %s already exists"
raise RuntimeError(msg % service)
@ -203,7 +200,7 @@ def subcommand_add_service(arguments):
# so add the service_on entry manually instead
path = convert_augeas_path_to_filepath(root)
with open(path, 'a') as servicefile:
line = "\nservice_on = %s\n" % convert_service_to_string(service)
line = "\nservice_on = %s\n" % utils.convert_service_to_string(service)
servicefile.write(line)
_service('restart')
@ -224,7 +221,7 @@ def get_new_service_path(protocol):
"""Get the augeas path of a new service for a protocol
This takes care of existing services using a /service_on/*/ query"""
root = get_augeas_servicefile_path(protocol)
root = utils.get_augeas_servicefile_path(protocol)
new_index = len(aug.match(root + '/*')) + 1
return os.path.join(root, str(new_index))
@ -233,8 +230,8 @@ def subcommand_get_kite(_):
"""Print details of the currently configured kite"""
kitename = aug.get(PATHS['kitename'])
kitesecret = aug.get(PATHS['kitesecret'])
print kitename if kitename else ''
print kitesecret if kitesecret else ''
print(kitename or '')
print(kitesecret or '')
def subcommand_set_kite(arguments):

View File

@ -1,113 +0,0 @@
#!/usr/bin/python2
# -*- mode: python -*-
#
# This file is part of Plinth.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
"""
Utilities for configuring PageKite.
"""
# TODO:
# Once python-augeas is available for python3 import the following things
# from plinth.modules.pagekite.util (instead of having a copy in here):
#
# SERVICE_PARAMS, convert_service_to_string
#
# until then, this file is python2 and python3 compatible for the unittests
import os
import json
CONF_PATH = '/files/etc/pagekite.d'
# parameters that get stored in configuration service_on entries
SERVICE_PARAMS = ['protocol', 'kitename', 'backend_host', 'backend_port',
'secret']
def convert_service_to_string(service):
""" Convert service dict into a ":"-separated parameter string
>>> convert_service_to_string({'kitename': '@kitename', \
'backend_host': 'localhost', 'secret': '@kitesecret', \
'protocol': 'https/443', 'backend_port': '443'})
'https/443:@kitename:localhost:443:@kitesecret'
"""
try:
service_string = ":".join([service[param] for param in SERVICE_PARAMS])
except KeyError:
raise ValueError("Could not parse params: %s " % service)
return service_string
def load_service(json_service):
""" create a service out of json command-line argument
1) parse json
2) only use the parameters that we need (SERVICE_PARAMS)
3) convert unicode to strings
"""
service = json.loads(json_service)
return dict((str(key), str(service[key])) for key in SERVICE_PARAMS)
def get_augeas_servicefile_path(protocol):
"""Get the augeas path where a service for a protocol should be stored
TODO: Once we use python3 switch from doctests to unittests
>>> get_augeas_servicefile_path('http')
'/files/etc/pagekite.d/80_http.rc/service_on'
>>> get_augeas_servicefile_path('https')
'/files/etc/pagekite.d/443_https.rc/service_on'
>>> get_augeas_servicefile_path('http/80')
'/files/etc/pagekite.d/80_http.rc/service_on'
>>> get_augeas_servicefile_path('http/8080')
'/files/etc/pagekite.d/8080_http.rc/service_on'
>>> get_augeas_servicefile_path('raw/22')
'/files/etc/pagekite.d/22_raw.rc/service_on'
>>> get_augeas_servicefile_path('xmpp')
Traceback (most recent call last):
...
ValueError: Unsupported protocol: xmpp
"""
if not protocol.startswith(("http", "https", "raw")):
raise ValueError('Unsupported protocol: %s' % protocol)
try:
_protocol, port = protocol.split('/')
except ValueError:
if protocol == 'http':
relpath = '80_http.rc'
elif protocol == 'https':
relpath = '443_https.rc'
else:
raise ValueError('Unsupported protocol: %s' % protocol)
else:
relpath = '%s_%s.rc' % (port, _protocol)
return os.path.join(CONF_PATH, relpath, 'service_on')
if __name__ == "__main__":
import doctest
doctest.testmod()

View File

@ -24,7 +24,7 @@ Configuration helper for Privoxy server.
import argparse
import re
import util
from plinth import action_utils
CONFIG_FILE = '/etc/privoxy/config'
@ -64,28 +64,28 @@ def subcommand_setup(_):
if not written:
conffile.write('listen-address [::]:8118')
util.service_restart('privoxy')
action_utils.service_restart('privoxy')
def subcommand_get_enabled(_):
"""Get whether service is enabled."""
is_enabled = util.service_is_enabled('privoxy')
is_enabled = action_utils.service_is_enabled('privoxy')
print('yes' if is_enabled else 'no')
def subcommand_enable(_):
"""Start service."""
util.service_enable('privoxy')
action_utils.service_enable('privoxy')
def subcommand_disable(_):
"""Stop service."""
util.service_disable('privoxy')
action_utils.service_disable('privoxy')
def subcommand_is_running(_):
"""Get whether server is running."""
running = util.service_is_running('privoxy')
running = action_utils.service_is_running('privoxy')
print('yes' if running else 'no')

View File

@ -25,7 +25,7 @@ import argparse
import os
import subprocess
import util
from plinth import action_utils
SERVICE_CONFIG = '/etc/default/tor'
TOR_CONFIG = '/etc/tor/torrc'
@ -59,7 +59,7 @@ def parse_arguments():
def subcommand_is_running(_):
"""Get whether Tor is running"""
print('yes' if util.service_is_running('tor') else 'no')
print('yes' if action_utils.service_is_running('tor') else 'no')
def subcommand_start(_):

View File

@ -25,8 +25,7 @@ from django.contrib import messages
from django.core import validators
from plinth.errors import ActionError
from .util import _run, get_kite_details, BACKEND_HOST, KITE_NAME, \
KITE_SECRET, PREDEFINED_SERVICES
from . import utils
LOGGER = logging.getLogger(__name__)
@ -75,28 +74,28 @@ for your account if no secret is set on the kite'))
if old['kite_name'] != new['kite_name'] or \
old['kite_secret'] != new['kite_secret']:
_run(['set-kite', '--kite-name', new['kite_name'],
'--kite-secret', new['kite_secret']])
utils.run(['set-kite', '--kite-name', new['kite_name'],
'--kite-secret', new['kite_secret']])
messages.success(request, _('Kite details set'))
config_changed = True
if old['server'] != new['server']:
_run(['set-frontend', new['server']])
utils.run(['set-frontend', new['server']])
messages.success(request, _('Pagekite server set'))
config_changed = True
if old['enabled'] != new['enabled']:
if new['enabled']:
_run(['start-and-enable'])
utils.run(['start-and-enable'])
messages.success(request, _('PageKite enabled'))
else:
_run(['stop-and-disable'])
utils.run(['stop-and-disable'])
messages.success(request, _('PageKite disabled'))
# Restart the service if the config was changed while the service
# was running, so changes take effect immediately.
elif config_changed and new['enabled']:
_run(['restart'])
utils.run(['restart'])
class StandardServiceForm(forms.Form):
@ -105,8 +104,8 @@ class StandardServiceForm(forms.Form):
def __init__(self, *args, **kwargs):
"""Add the fields from PREDEFINED_SERVICES"""
super(StandardServiceForm, self).__init__(*args, **kwargs)
kite = get_kite_details()
for name, service in PREDEFINED_SERVICES.items():
kite = utils.get_kite_details()
for name, service in utils.PREDEFINED_SERVICES.items():
if name in ('http', 'https'):
help_text = service['help_text'].format(kite['kite_name'])
else:
@ -117,16 +116,16 @@ class StandardServiceForm(forms.Form):
def save(self, request):
formdata = self.cleaned_data
for service_name in PREDEFINED_SERVICES.keys():
for service_name in utils.PREDEFINED_SERVICES.keys():
if self.initial[service_name] != formdata[service_name]:
service = PREDEFINED_SERVICES[service_name]['params']
service = utils.PREDEFINED_SERVICES[service_name]['params']
service = json.dumps(service)
if formdata[service_name]:
_run(['add-service', '--service', service])
utils.run(['add-service', '--service', service])
messages.success(request, _('Service enabled: {name}')
.format(name=service_name))
else:
_run(['remove-service', '--service', service])
utils.run(['remove-service', '--service', service])
messages.success(request, _('Service disabled: {name}')
.format(name=service_name))
@ -151,11 +150,11 @@ class BaseCustomServiceForm(forms.Form):
# set kitename and kitesecret if not already set
if 'kitename' not in formdata:
if 'subdomains' in formdata and formdata['subdomains']:
formdata['kitename'] = "*.%s" % KITE_NAME
formdata['kitename'] = "*.%s" % utils.KITE_NAME
else:
formdata['kitename'] = KITE_NAME
formdata['kitename'] = utils.KITE_NAME
if 'secret' not in formdata:
formdata['secret'] = KITE_SECRET
formdata['secret'] = utils.KITE_SECRET
# merge protocol and frontend_port back to one entry (protocol)
if 'frontend_port' in formdata:
@ -163,7 +162,7 @@ class BaseCustomServiceForm(forms.Form):
formdata['protocol'] = "%s/%s" % (formdata['protocol'],
formdata['frontend_port'])
if 'backend_host' not in formdata:
formdata['backend_host'] = BACKEND_HOST
formdata['backend_host'] = utils.BACKEND_HOST
return formdata
@ -172,7 +171,7 @@ class DeleteCustomServiceForm(BaseCustomServiceForm):
def delete(self, request):
service = self.convert_formdata_to_service(self.cleaned_data)
_run(['remove-service', '--service', json.dumps(service)])
utils.run(['remove-service', '--service', json.dumps(service)])
messages.success(request, _('Deleted custom service'))
@ -183,7 +182,7 @@ class AddCustomServiceForm(BaseCustomServiceForm):
"""Returns whether the user input matches a predefined service"""
service = self.convert_formdata_to_service(formdata)
match_found = False
for predefined_service_obj in PREDEFINED_SERVICES.values():
for predefined_service_obj in utils.PREDEFINED_SERVICES.values():
# manually add the port to compare predefined with custom services
# that's due to the (sometimes) implicit port in the configuration
predefined_service = copy.copy(predefined_service_obj['params'])
@ -215,7 +214,7 @@ class AddCustomServiceForm(BaseCustomServiceForm):
def save(self, request):
service = self.convert_formdata_to_service(self.cleaned_data)
try:
_run(['add-service', '--service', json.dumps(service)])
utils.run(['add-service', '--service', json.dumps(service)])
messages.success(request, _('Added custom service'))
except ActionError as exception:
if "already exists" in str(exception):

View File

@ -16,7 +16,7 @@
#
from django import template
from plinth.modules.pagekite.util import prepare_service_for_display
from plinth.modules.pagekite import utils
register = template.Library()
@ -31,7 +31,7 @@ def create_pagekite_service_url(service, kite_name):
"""
# add extra information if it's missing
if 'subdomains' not in service:
service = prepare_service_for_display(service)
service = utils.prepare_service_for_display(service)
urlparams = {'protocol': service['protocol']}
if service['subdomains']:

View File

@ -21,6 +21,8 @@ Test modules for Pagekite functions.
import unittest
from plinth.modules.pagekite import utils
class TestPagekiteActions(unittest.TestCase):
"""Test-cases for the pagekite action utils"""
@ -49,10 +51,8 @@ class TestPagekiteActions(unittest.TestCase):
},
]
@unittest.skip('Use this test once the function is in the pagekite module '
'instead of actions/pagekite_util.py')
def test_convert_service_to_string(self):
""" Test deconstructing parameter dictionaries into strings """
for test in self._tests:
service_string = convert_service_to_string(test['params'])
service_string = utils.convert_service_to_string(test['params'])
self.assertEqual(test['line'], service_string)

View File

@ -18,6 +18,7 @@
from gettext import gettext as _
import json
import logging
import os
from plinth import actions
@ -29,6 +30,10 @@ BACKEND_HOST = 'localhost'
KITE_NAME = '@kitename'
KITE_SECRET = '@kitesecret'
# Augeas base path for Pagekite configuration files
CONF_PATH = '/files/etc/pagekite.d'
# Parameters that get stored in configuration service_on entries
SERVICE_PARAMS = ['protocol', 'kitename', 'backend_host', 'backend_port',
'secret']
@ -72,7 +77,7 @@ PREDEFINED_SERVICES = {
def get_kite_details():
output = _run(['get-kite'])
output = run(['get-kite'])
kite_details = output.split()
return {'kite_name': kite_details[0],
'kite_secret': kite_details[1]}
@ -86,14 +91,14 @@ def get_pagekite_config():
# PageKite service enabled/disabled
# This assumes that if pagekite is running it's also enabled as a service
output = _run(['is-running'])
output = run(['is-running'])
status['enabled'] = (output.split()[0] == 'yes')
# PageKite kite details
status.update(get_kite_details())
# PageKite frontend server
server = _run(['get-frontend'])
server = run(['get-frontend'])
status['server'] = server.replace('\n', '')
return status
@ -110,7 +115,7 @@ def get_pagekite_services():
# set all predefined services to 'disabled' by default
[predefined.update({proto: False}) for proto in PREDEFINED_SERVICES.keys()]
# now, search for the enabled ones
for serviceline in _run(['get-services']).split('\n'):
for serviceline in run(['get-services']).split('\n'):
if not serviceline: # skip empty lines
continue
@ -137,7 +142,7 @@ def prepare_service_for_display(service):
return service
def _run(arguments, superuser=True):
def run(arguments, superuser=True):
"""Run a given command and raise exception if there was an error"""
command = 'pagekite'
@ -145,3 +150,78 @@ def _run(arguments, superuser=True):
return actions.superuser_run(command, arguments)
else:
return actions.run(command, arguments)
def convert_service_to_string(service):
""" Convert service dict into a ":"-separated parameter string
>>> convert_service_to_string({'kitename': '@kitename', \
'backend_host': 'localhost', 'secret': '@kitesecret', \
'protocol': 'https/443', 'backend_port': '443'})
'https/443:@kitename:localhost:443:@kitesecret'
"""
try:
service_string = ":".join([service[param] for param in SERVICE_PARAMS])
except KeyError:
raise ValueError("Could not parse params: %s " % service)
return service_string
def load_service(json_service):
""" create a service out of json command-line argument
1) parse json
2) only use the parameters that we need (SERVICE_PARAMS)
3) convert unicode to strings
"""
service = json.loads(json_service)
return dict((str(key), str(service[key])) for key in SERVICE_PARAMS)
def get_augeas_servicefile_path(protocol):
"""Get the augeas path where a service for a protocol should be stored
TODO: Once we use python3 switch from doctests to unittests
>>> get_augeas_servicefile_path('http')
'/files/etc/pagekite.d/80_http.rc/service_on'
>>> get_augeas_servicefile_path('https')
'/files/etc/pagekite.d/443_https.rc/service_on'
>>> get_augeas_servicefile_path('http/80')
'/files/etc/pagekite.d/80_http.rc/service_on'
>>> get_augeas_servicefile_path('http/8080')
'/files/etc/pagekite.d/8080_http.rc/service_on'
>>> get_augeas_servicefile_path('raw/22')
'/files/etc/pagekite.d/22_raw.rc/service_on'
>>> get_augeas_servicefile_path('xmpp')
Traceback (most recent call last):
...
ValueError: Unsupported protocol: xmpp
"""
if not protocol.startswith(("http", "https", "raw")):
raise ValueError('Unsupported protocol: %s' % protocol)
try:
_protocol, port = protocol.split('/')
except ValueError:
if protocol == 'http':
relpath = '80_http.rc'
elif protocol == 'https':
relpath = '443_https.rc'
else:
raise ValueError('Unsupported protocol: %s' % protocol)
else:
relpath = '%s_%s.rc' % (port, _protocol)
return os.path.join(CONF_PATH, relpath, 'service_on')
if __name__ == "__main__":
import doctest
doctest.testmod()

View File

@ -24,8 +24,7 @@ from django.views.generic import View, TemplateView
from django.views.generic.edit import FormView
from plinth import package
from .util import get_pagekite_config, get_pagekite_services, \
get_kite_details, prepare_service_for_display
from . import utils
from .forms import ConfigurationForm, StandardServiceForm, \
AddCustomServiceForm, DeleteCustomServiceForm
@ -79,12 +78,12 @@ class CustomServiceView(ContextMixin, TemplateView):
def get_context_data(self, *args, **kwargs):
context = super(CustomServiceView, self).get_context_data(*args,
**kwargs)
unused, custom_services = get_pagekite_services()
unused, custom_services = utils.get_pagekite_services()
for service in custom_services:
service['form'] = AddCustomServiceForm(initial=service)
context['custom_services'] = [prepare_service_for_display(service)
context['custom_services'] = [utils.prepare_service_for_display(service)
for service in custom_services]
context.update(get_kite_details())
context.update(utils.get_kite_details())
return context
def get(self, request, *args, **kwargs):
@ -112,7 +111,7 @@ class StandardServiceView(ContextMixin, FormView):
success_url = reverse_lazy('pagekite:standard-services')
def get_initial(self):
return get_pagekite_services()[0]
return utils.get_pagekite_services()[0]
def form_valid(self, form):
form.save(self.request)
@ -126,7 +125,7 @@ class ConfigurationView(ContextMixin, FormView):
success_url = reverse_lazy('pagekite:configure')
def get_initial(self):
return get_pagekite_config()
return utils.get_pagekite_config()
def form_valid(self, form):
form.save(self.request)

View File

@ -2,4 +2,5 @@ cherrypy >= 3.0
coverage >= 3.7
django >= 1.7.0
django-stronghold
python-augeas
pyyaml

View File

@ -116,6 +116,7 @@ setuptools.setup(
'django >= 1.7.0',
'django-bootstrap-form',
'django-stronghold',
'python-augeas',
'pyyaml',
],
tests_require=['coverage >= 3.7'],