Merge branch 'issue-535'

neoexchange/astrometrics/sources_subs.py

@@ -24,12 +24,14 @@
 import email
 from re import sub, compile
 from math import degrees
-from datetime import datetime, timedelta
+from time import sleep
+from datetime import date, datetime, timedelta
 from socket import error
 from random import randint
-from time import sleep
-from datetime import date
 import requests
+import shutil
+import tempfile
+from contextlib import closing
 
 from bs4 import BeautifulSoup
 import astropy.units as u
@@ -1448,8 +1450,29 @@ def get_site_status(site_code):
     return good_to_schedule, reason
 
 
-def fetch_yarkovsky_targets(yark_targets):
-    """Fetches Yarkovsky targets from command line and returns a list of targets"""
+def fetch_yarkovsky_targets(targets_or_file=None):
+    """Main wrapper routine for either fetch_yarkovsky_targets_list() or
+    fetch_yarkovsky_targets_ftp() to fetch Yarkovsky targets.
+    If [targets_or_file] is a `list` of targets, fetch_yarkovsky_targets_list()
+    is called; if [targets_or_file] is a filename or None, then
+    fetch_yarkovsky_targets_ftp() is called and the target list comes from either
+    the FTP site (`targets_or_file=None`) or by reading the file specified by
+    `targets_or_file`.
+
+    Returns a list of target names.
+    """
+
+    if type(targets_or_file) == list:
+        yark_target_list = fetch_yarkovsky_targets_list(targets_or_file)
+    else:
+        yark_target_list = fetch_yarkovsky_targets_ftp(targets_or_file)
+
+    return yark_target_list
+
+
+def fetch_yarkovsky_targets_list(yark_targets):
+    """Parses a list of lines of Yarkovsky targets (read from a file and which
+    may contain comments) and returns a list of targets"""
 
     yark_target_list = []
 
@@ -1466,6 +1489,45 @@ def fetch_yarkovsky_targets(yark_targets):
     return yark_target_list
 
 
+def fetch_yarkovsky_targets_ftp(file_or_url=None):
+    """Fetches Yarkovsky targets from either the specified file (if [file_or_url]
+    is not None) or the current list from the FTP site
+    and parses it to return the target and expected A2 Yarkovsky value and
+    its error"""
+    ftp_url = 'ftp://ssd.jpl.nasa.gov/pub/ssd/yarkovsky/yarko_targets/yarko_latest.txt'
+
+    targets = []
+    tempdir = None
+
+    if file_or_url is None:
+        tempdir = tempfile.mkdtemp(prefix='tmp_neox_')
+        target_file = os.path.join(tempdir, 'yarkovsky_targets.txt')
+
+        with closing(urllib.request.urlopen(ftp_url)) as read_fp:
+            with open(target_file, 'wb') as write_fp:
+                shutil.copyfileobj(read_fp, write_fp)
+    else:
+        target_file = file_or_url
+
+    if os.path.exists(target_file):
+        table = ascii.read(target_file, format='csv')
+
+        for target in list(table['base']):
+            target = target.upper()
+            # Look for designations of the for yyyyXY[12] and add space in the middle
+            if len(target) >=6 and target[0:4].isdigit() and target[4:6].isalpha():
+                target = target[0:4] + ' ' + target[4:]
+            targets.append(target)
+
+        if tempdir:
+            try:
+                os.remove(target_file)
+                os.rmdir(tempdir)
+            except FileNotFoundError:
+                pass
+
+    return targets
+
 def fetch_sfu(page=None):
     """Fetches the solar radio flux from the Solar Radio Monitoring
     Program run by National Research Council and Natural Resources Canada.

neoexchange/astrometrics/tests/test_sources_subs.py

@@ -24,7 +24,7 @@
 
 import astropy.units as u
 from bs4 import BeautifulSoup
-from django.test import TestCase
+from django.test import TestCase, SimpleTestCase
 from django.forms.models import model_to_dict
 
 from core.models import Body, Proposal, Block, StaticSource, PhysicalParameters, Designations, ColorValues
@@ -619,7 +619,10 @@ def test_target_with_ampersand4(self):
         self.assertEqual(expected_target, targets)
 
 
-class TestFetchYarkovskyTargets(TestCase):
+class TestFetchYarkovskyTargets(SimpleTestCase):
+
+    def setUp(self):
+        self.test_file = os.path.abspath(os.path.join('astrometrics', 'tests', 'test_yarkovsky_targets.txt'))
 
     def test_read_from_file(self):
         expected_targets = [ '1999 NW2',
@@ -657,6 +660,44 @@ def test_read_from_commandline(self):
 
         self.assertEqual(expected_targets, target_list)
 
+    def test_fetch_from_ftp(self):
+        expected_targets = [ '433',
+                             '467352',
+                             '2002 TS69',
+                             '401856',
+                             '2011 JY1',
+                             '1998 WB2',
+                             '2015 KJ19',
+                             '2003 MK4',
+                             '2003 GQ22']
+
+
+        targets = fetch_yarkovsky_targets(self.test_file)
+
+        self.assertEqual(expected_targets, targets)
+
+
+class TestFetchYarkovskyTargetsFTP(SimpleTestCase):
+
+    def setUp(self):
+        self.test_file = os.path.abspath(os.path.join('astrometrics', 'tests', 'test_yarkovsky_targets.txt'))
+
+    def test_fetch_latest(self):
+        expected_targets = [ '433',
+                             '467352',
+                             '2002 TS69',
+                             '401856',
+                             '2011 JY1',
+                             '1998 WB2',
+                             '2015 KJ19',
+                             '2003 MK4',
+                             '2003 GQ22']
+
+
+        targets = fetch_yarkovsky_targets_ftp(self.test_file)
+
+        self.assertEqual(expected_targets, targets)
+
 
 class TestSubmitBlockToScheduler(TestCase):
     """Also tests make_requestgroup()"""

neoexchange/astrometrics/tests/test_yarkovsky_targets.txt

@@ -0,0 +1,10 @@
+base, A2_exp, sig_A2, sig_A2_sim, SNR_exp, improvement
+433, 8.364075e-14, 3.471314e-14, 7.219550e-15, 11.585313, 4.808214
+467352, 8.364075e-14, 3.471314e-14, 7.219550e-15, 11.585313, 4.808214
+2002ts69, 5.039853e-13, 1.457149e-12, 7.604491e-14, 6.627470, 19.161690
+401856, 7.284804e-14, 3.275321e-14, 1.221453e-14, 5.964048, 2.681496
+2011jy1, 5.526089e-13, 2.001942e-11, 9.725019e-14, 5.682343, 205.854816
+1998wb2, 1.522011e-13, 6.015808e-12, 2.877142e-14, 5.290011, 209.089715
+2015kj19, 2.006402e-13, 3.098212e-12, 4.871464e-14, 4.118683, 63.599197
+2003mk4, 1.005583e-13, 2.915890e-13, 2.865736e-14, 3.508987, 10.175013
+2003gq22, 1.154564e-13, 9.387852e-14, 3.587755e-14, 3.218067, 2.616637

neoexchange/core/management/commands/compute_long_term_scheduling.py

@@ -33,7 +33,7 @@ def add_arguments(self, parser):
         parser.add_argument('site_code', help='MPC site code')
         parser.add_argument('dark_and_up_time_limit', type=float, help='Amount of time sky must be dark and target is above the horizon')
         parser.add_argument('targets', nargs='*', help='Targets to schedule')
-        parser.add_argument('--targetlist', action="store", default=None, help="File of targets to read (optional)")
+        parser.add_argument('--targetlist', action="store", default=None, help="File of targets to read (optional; set to 'FTP' to read from JPL site)")
         parser.add_argument('--start_date', default=datetime.utcnow().strftime('%Y-%m-%d'), help='Date to start ephemeris search in YYYY-MM-DD format')
         parser.add_argument('--date_range', type=int, default=30, help='Date range ephemeris search in days')
 
@@ -42,10 +42,17 @@ def handle(self, *args, **options):
         self.stdout.write("========================")
         targets = []
         if options['targetlist'] is not None:
-            with open(expanduser(options['targetlist'])) as f:
-                targets = f.readlines()
-        targets += options['targets']
-        target_list = fetch_yarkovsky_targets(targets)
+            if options['targetlist'] == 'FTP':
+                targets = None
+            else:
+                with open(expanduser(options['targetlist'])) as f:
+                    targets = f.readlines()
+
+            target_list = fetch_yarkovsky_targets(targets)
+        target_list += options['targets']
+        self.stdout.write("Combined target list")
+        self.stdout.write("\n".join(target_list))
+        self.stdout.write("========================")
         for obj_id in target_list:
             orbelems = model_to_dict(Body.objects.get(name=obj_id))
             visible_dates, emp_visible_dates, dark_and_up_time_all, max_alt_all = monitor_long_term_scheduling(options['site_code'], orbelems, datetime.strptime(options['start_date'], '%Y-%m-%d'), options['date_range'], options['dark_and_up_time_limit'])

neoexchange/core/management/commands/fetch_yarkovsky_targets.py

@@ -27,18 +27,22 @@ class Command(BaseCommand):
     help = 'Fetch Yarkovsky target list for the current month'
 
     def add_arguments(self, parser):
-        parser.add_argument('--targetlist', action="store", default=None, help="File of targets to read (optional)")
+        parser.add_argument('--targetlist', action="store", default=None, help="File of targets to read (optional; set to 'FTP' to read from JPL site)")
         parser.add_argument('yark_targets', nargs='*', help='List of Yarkovsky targets to ingest')
 
     def handle(self, *args, **options):
         self.stdout.write("==== Fetching Yarkovsky targets %s ====" % (datetime.now().strftime('%Y-%m-%d %H:%M')))
 
         targets = []
         if options['targetlist'] is not None:
-            with open(expanduser(options['targetlist'])) as f:
-                targets = f.readlines()
-        targets += options['yark_targets']
-        yark_targets = fetch_yarkovsky_targets(targets)
+            if options['targetlist'] == 'FTP':
+                targets = None
+            else:
+                with open(expanduser(options['targetlist'])) as f:
+                    targets = f.readlines()
+
+            yark_targets = fetch_yarkovsky_targets(targets)
+        yark_targets += options['yark_targets']
         for obj_id in yark_targets:
             self.stdout.write("Reading Yarkovsky target %s" % obj_id)
             update_MPC_orbit(obj_id, origin='Y')