Enable support for PUNCH CAM/PAM images

docroot/schema/image_layer.schema.json

@@ -38,7 +38,7 @@
         "nickname": {
             "type": "string",
             "enum": [
-                "GOES-R SUVI 94","GOES-R SUVI 131","GOES-R SUVI 171","GOES-R SUVI 195","GOES-R SUVI 284","GOES-R SUVI 304","EIT 171","EIT 195","EIT 284","EIT 304","LASCO C2","LASCO C3","MDI Mag","MDI Int","AIA 94","AIA 131","AIA 171","AIA 193","AIA 211","AIA 304","AIA 335","AIA 1600","AIA 1700","AIA 4500","HMI Int","HMI Mag","EUVI-A 171","EUVI-A 195","EUVI-A 284","EUVI-A 304","EUVI-B 171","EUVI-B 195","EUVI-B 284","EUVI-B 304","COR1-A","COR2-A","COR1-B","COR2-B","SWAP 174","SXT AlMgMn","SXT thin-Al","SXT white-light","XRT Al_med/Al_mesh","XRT Al_med/Al_thick","XRT Al_med/Be_thick","XRT Al_med/Gband","XRT Al_med/Open","XRT Al_med/Ti_poly","XRT Al_poly/Al_mesh","XRT Al_poly/Al_thick","XRT Al_poly/Be_thick","XRT Al_poly/Gband","XRT Al_poly/Open","XRT Al_poly/Ti_poly","XRT Be_med/Al_mesh","XRT Be_med/Al_thick","XRT Be_med/Be_thick","XRT Be_med/Gband","XRT Be_med/Open","XRT Be_med/Ti_poly","XRT Be_thin/Al_mesh","XRT Be_thin/Al_thick","XRT Be_thin/Be_thick","XRT Be_thin/Gband","XRT Be_thin/Open","XRT Be_thin/Ti_poly","XRT C_poly/Al_mesh","XRT C_poly/Al_thick","XRT C_poly/Be_thick","XRT C_poly/Gband","XRT C_poly/Open","XRT C_poly/Ti_poly","XRT Mispositioned/Mispositioned","XRT Open/Al_mesh","XRT Open/Al_thick","XRT Open/Be_thick","XRT Open/Gband","XRT Open/Open","XRT Open/Ti_poly","TRACE 171","TRACE 195","TRACE 284","TRACE 1216","TRACE 1550","TRACE 1600","TRACE 1700","TRACE white-light","COSMO KCor","EUI FSI 174","EUI FSI 304","EUI HRI 174","EUI HRI 1216","IRIS SJI 1330","IRIS SJI 2796","IRIS SJI 1400","IRIS SJI 1600","IRIS SJI 2832","IRIS SJI 5000","GONG H-alpha","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","PUNCH","GOES CCOR-1","SWFO-L1 CCOR-2","SUVI 94","SUVI 131","SUVI 171","SUVI 195","SUVI 284","SUVI 304","XRT Any/Any","XRT Any/Al_mesh","XRT Any/Al_thick","XRT Any/Be_thick","XRT Any/Gband","XRT Any/Open","XRT Any/Ti_poly","XRT Al_med/Any","XRT Al_poly/Any","XRT Be_med/Any","XRT Be_thin/Any","XRT C_poly/Any","XRT Open/Any"
+                "GOES-R SUVI 94","GOES-R SUVI 131","GOES-R SUVI 171","GOES-R SUVI 195","GOES-R SUVI 284","GOES-R SUVI 304","EIT 171","EIT 195","EIT 284","EIT 304","LASCO C2","LASCO C3","MDI Mag","MDI Int","AIA 94","AIA 131","AIA 171","AIA 193","AIA 211","AIA 304","AIA 335","AIA 1600","AIA 1700","AIA 4500","HMI Int","HMI Mag","EUVI-A 171","EUVI-A 195","EUVI-A 284","EUVI-A 304","EUVI-B 171","EUVI-B 195","EUVI-B 284","EUVI-B 304","COR1-A","COR2-A","COR1-B","COR2-B","SWAP 174","SXT AlMgMn","SXT thin-Al","SXT white-light","XRT Al_med/Al_mesh","XRT Al_med/Al_thick","XRT Al_med/Be_thick","XRT Al_med/Gband","XRT Al_med/Open","XRT Al_med/Ti_poly","XRT Al_poly/Al_mesh","XRT Al_poly/Al_thick","XRT Al_poly/Be_thick","XRT Al_poly/Gband","XRT Al_poly/Open","XRT Al_poly/Ti_poly","XRT Be_med/Al_mesh","XRT Be_med/Al_thick","XRT Be_med/Be_thick","XRT Be_med/Gband","XRT Be_med/Open","XRT Be_med/Ti_poly","XRT Be_thin/Al_mesh","XRT Be_thin/Al_thick","XRT Be_thin/Be_thick","XRT Be_thin/Gband","XRT Be_thin/Open","XRT Be_thin/Ti_poly","XRT C_poly/Al_mesh","XRT C_poly/Al_thick","XRT C_poly/Be_thick","XRT C_poly/Gband","XRT C_poly/Open","XRT C_poly/Ti_poly","XRT Mispositioned/Mispositioned","XRT Open/Al_mesh","XRT Open/Al_thick","XRT Open/Be_thick","XRT Open/Gband","XRT Open/Open","XRT Open/Ti_poly","TRACE 171","TRACE 195","TRACE 284","TRACE 1216","TRACE 1550","TRACE 1600","TRACE 1700","TRACE white-light","COSMO KCor","EUI FSI 174","EUI FSI 304","EUI HRI 174","EUI HRI 1216","IRIS SJI 1330","IRIS SJI 2796","IRIS SJI 1400","IRIS SJI 1600","IRIS SJI 2832","IRIS SJI 5000","GONG H-alpha","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","RHESSI","PUNCH","GOES","SWFO-L1","PUNCH","GOES-R SUVI 94","GOES-R SUVI 131","GOES-R SUVI 171","GOES-R SUVI 195","GOES-R SUVI 284","GOES-R SUVI 304","XRT Any/Any","XRT Any/Al_mesh","XRT Any/Al_thick","XRT Any/Be_thick","XRT Any/Gband","XRT Any/Open","XRT Any/Ti_poly","XRT Al_med/Any","XRT Al_poly/Any","XRT Be_med/Any","XRT Be_thin/Any","XRT C_poly/Any","XRT Open/Any"
             ]
         },
         "sourceId": { "type": "integer" },
@@ -52,14 +52,14 @@
                     "name": {
                         "type": "string",
                         "enum": [
-                            "", "GOES-R", "100-300keV","12-25keV","1216","131","1330","1400","1550","1600","1700","171","174","193","195","211","25-50keV","2796","2832","284","3-6keV","304","335","4500","50-100keV","5000","530","6-12keV","6562","735","94","AIA","AlMgMn","Al_med","Al_mesh","Al_poly","Al_thick","Any","Back Projection","Be_med","Be_thick","Be_thin","C2","C3","CCOR-1","CCOR-2","Clean","Clean 59","continuum","COR1","COR2","COSMO","C_poly","EIT","EUI","EUVI","FSI","Gband","GOES","GONG","H-alpha","Hinode","HMI","HRI","IRIS","KCor","LASCO","magnetogram","MDI","MEM GE","Mispositioned","MLSO","Open","PROBA2","PUNCH","RHESSI","SDO","SECCHI","SJI","SOHO","SOLO","STEREO_A","STEREO_B","SUVI","SWAP","SWFO-L1","SXT","thin-Al","Ti_poly","TRACE","VIS CS","VIS FWDFIT","WFI+NFI","white-light","XRT","Yohkoh"
+                            "", "GOES-R", "100-300keV","12-25keV","1216","131","1330","1400","1550","1600","1700","171","174","193","195","211","25-50keV","2796","2832","284","3-6keV","304","335","4500","50-100keV","5000","6-12keV","6562","735","94","AIA","Al_med","Al_mesh","Al_poly","Al_thick","AlMgMn","Any","Back Projection","Be_med","Be_thick","Be_thin","C_poly","C2","C3","CCOR-1","CCOR-2","Clean","Clean 59","continuum","COR1","COR2","COSMO","EIT","EUI","EUVI","FSI","Gband","GOES","GONG","H-alpha","Hinode","HMI","HRI","IRIS","KCor","LASCO","magnetogram","MDI","MEM GE","Mispositioned","MLSO","Open","Polarized Brightness","PROBA2","PUNCH","RHESSI","SDO","SECCHI","SJI","SOHO","SOLO","STEREO_A","STEREO_B","SUVI","SWAP","SWFO-L1","SXT","thin-Al","Ti_poly","Total Brightness","TRACE","VIS CS","VIS FWDFIT","WFI+NFI","white-light","XRT","Yohkoh"
                         ],
                         "$comment": "Autogenerated from available sources"
                     },
                     "label": {
                         "type": "string",
                         "enum": [
-                          "", "Detector","Energy Band","Filter","Filter Wheel 1","Filter Wheel 2","Instrument","Measurement","Observatory","Reconstruction"
+                            "","Detector","Energy Band","Filter","Filter Wheel 1","Filter Wheel 2","Instrument","Measurement","Observatory","Reconstruction"
                         ],
                         "$comment": "Autogenerated from available sources"
                     }
@@ -81,7 +81,7 @@
         "^(Detector|Energy Band|Filter|Filter Wheel 1|Filter Wheel 2|Instrument|Measurement|Observatory|Reconstruction)$": {
             "type": "string",
             "enum": [
-                "GOES-R", "100-300keV","12-25keV","1216","131","1330","1400","1550","1600","1700","171","174","193","195","211","25-50keV","2796","2832","284","3-6keV","304","335","4500","50-100keV","5000","530","6-12keV","6562","735","94","AIA","AlMgMn","Al_med","Al_mesh","Al_poly","Al_thick","Any","Back Projection","Be_med","Be_thick","Be_thin","C2","C3","CCOR-1","CCOR-2","Clean","Clean 59","continuum","COR1","COR2","COSMO","C_poly","EIT","EUI","EUVI","FSI","Gband","GOES","GONG","H-alpha","Hinode","HMI","HRI","IRIS","KCor","LASCO","magnetogram","MDI","MEM GE","Mispositioned","MLSO","Open","PROBA2","PUNCH","RHESSI","SDO","SECCHI","SJI","SOHO","SOLO","STEREO_A","STEREO_B","SUVI","SWAP","SWFO-L1","SXT","thin-Al","Ti_poly","TRACE","VIS CS","VIS FWDFIT","WFI+NFI","white-light","XRT","Yohkoh"
+                "GOES-R", "100-300keV","12-25keV","1216","131","1330","1400","1550","1600","1700","171","174","193","195","211","25-50keV","2796","2832","284","3-6keV","304","335","4500","50-100keV","5000","6-12keV","6562","735","94","AIA","Al_med","Al_mesh","Al_poly","Al_thick","AlMgMn","Any","Back Projection","Be_med","Be_thick","Be_thin","C_poly","C2","C3","CCOR-1","CCOR-2","Clean","Clean 59","continuum","COR1","COR2","COSMO","EIT","EUI","EUVI","FSI","Gband","GOES","GONG","H-alpha","Hinode","HMI","HRI","IRIS","KCor","LASCO","magnetogram","MDI","MEM GE","Mispositioned","MLSO","Open","Polarized Brightness","PROBA2","PUNCH","RHESSI","SDO","SECCHI","SJI","SOHO","SOLO","STEREO_A","STEREO_B","SUVI","SWAP","SWFO-L1","SXT","thin-Al","Ti_poly","Total Brightness","TRACE","VIS CS","VIS FWDFIT","WFI+NFI","white-light","XRT","Yohkoh"
             ]
         }
     }

docroot/schema/post_movie.schema.json

@@ -17,7 +17,7 @@
         "layers": {
             "type": "string",
             "maxLength": 1000,
-            "pattern": "^[\\_\\w\\]\\[,\\-:\\.]+$"
+            "pattern": "^[\\_\\w\\]\\[,\\-:\\.\\+ ]+$"
         },
         "eventsState": {
             "$ref": "https://api.helioviewer.org/schema/event_layers.schema.json"

docroot/schema/post_screenshot.schema.json

@@ -15,7 +15,7 @@
         "layers": {
             "type": "string",
             "maxLength": 1000,
-            "pattern": "^[\\_\\w\\]\\[,\\-:\\.]+$"
+            "pattern": "^[\\_\\w\\]\\[,\\-:\\.\\+ ]+$"
         },
         "eventsState": {
             "$ref": "https://api.helioviewer.org/schema/event_layers.schema.json"

install/__test__/test_punch.py

@@ -0,0 +1,25 @@
+import unittest
+import datetime
+from helioviewer.hvpull.servers.punch import PUNCHDataServer
+
+class TestPunchDataServer(unittest.TestCase):
+  def test_compute_directories(self):
+    punch = PUNCHDataServer()
+    dirs = punch.compute_directories(datetime.datetime(2026, 1, 1), datetime.datetime(2026, 1, 5))
+    assert dirs == [
+      'https://umbra.nascom.nasa.gov/punch/L/3/CAM/2026/01/05',
+      'https://umbra.nascom.nasa.gov/punch/L/3/PAM/2026/01/05',
+      'https://umbra.nascom.nasa.gov/punch/L/3/CAM/2026/01/04',
+      'https://umbra.nascom.nasa.gov/punch/L/3/PAM/2026/01/04',
+      'https://umbra.nascom.nasa.gov/punch/L/3/CAM/2026/01/03',
+      'https://umbra.nascom.nasa.gov/punch/L/3/PAM/2026/01/03',
+      'https://umbra.nascom.nasa.gov/punch/L/3/CAM/2026/01/02',
+      'https://umbra.nascom.nasa.gov/punch/L/3/PAM/2026/01/02',
+      'https://umbra.nascom.nasa.gov/punch/L/3/CAM/2026/01/01',
+      'https://umbra.nascom.nasa.gov/punch/L/3/PAM/2026/01/01',
+    ]
+
+  def test_get_datetime_from_file(self):
+    punch = PUNCHDataServer()
+    assert punch.get_datetime_from_file("PUNCH_L3_PAM_20251102001600_v0j.fits") == datetime.datetime(2025, 11, 2, 0, 16, 0)
+    assert punch.get_datetime_from_file("PUNCH_L3_CAM_20260221001600_v0j.fits") == datetime.datetime(2026, 2, 21, 0, 16, 0)

install/database/2026_04_07_punch_rename.sql

@@ -0,0 +1,14 @@
+UPDATE datasources SET description = 'PUNCH WFI+NFI CAM Mosaic' WHERE id = 131;
+
+UPDATE datasource_property SET name = 'WFI+NFI', fitsName = 'WFI+NFI', description = 'Clear low-noise science mosaic, bkg-sub & resolved into B & uncertainty layer' WHERE sourceId = 131 AND label = 'Instrument';
+
+UPDATE datasource_property SET name = 'Total Brightness', fitsName = 'PUNCH Level-3 Intermediate F-corona Subtracted Unpolarized Mosaic' WHERE sourceId = 131 AND label = 'Measurement';
+
+INSERT INTO datasources (id, name, description, units, layeringOrder, enabled, sourceIdGroup, displayOrder)
+VALUES (134, 'PUNCH', 'PUNCH WFI+NFI PAM Mosaic', NULL, 1, 0, '', 0);
+
+INSERT INTO datasource_property (sourceId, label, name, fitsName, description, uiOrder)
+VALUES
+(134, 'Observatory', 'PUNCH', 'PUNCH', 'PUNCH', 1),
+(134, 'Instrument', 'WFI+NFI', 'WFI+NFI', 'Polarized low-noise science mosaic, bkg-sub & resolved into B, pB, & uncertainty layer', 2),
+(134, 'Measurement', 'Polarized Brightness', 'PUNCH Level-3 Intermediate F-corona Subtracted Polarized Mosaic', '', 3);

install/helioviewer/db.py

@@ -361,7 +361,8 @@ def create_datasource_table(cursor):
 (128, 'RHESSI', 'RHESSI 25-50keV VIS FWDFIT',    NULL, 1, 0, '', 0, 0, 0, 0),
 (129, 'RHESSI', 'RHESSI 50-100keV VIS FWDFIT',   NULL, 1, 0, '', 0, 0, 0, 0),
 (130, 'RHESSI', 'RHESSI 100-300keV VIS FWDFIT',  NULL, 1, 0, '', 0, 0, 0, 0),
-(131,  'PUNCH', 'PUNCH WFI+NFI', NULL, 1, 0, '', 0, 0, 0, 0),
+(131,  'PUNCH', 'PUNCH WFI+NFI CAM Mosaic', NULL, 1, 0, '', 0, 0, 0, 0),
+(134,  'PUNCH', 'PUNCH WFI+NFI PAM Mosaic', NULL, 1, 0, '', 0, 0, 0, 0),
 (132, 'GOES', 'CCOR-1', NULL, 1, 0, '', 0, 0, 0, 0),
 (133, 'SWFO-L1', 'CCOR-2',  NULL, 1, 0, '', 0, 0, 0, 0),
 (2000, 'GOES-R SUVI 94', 'GOES-R SUVI 94',  NULL, 1, 0, '', 0, 0, 0, 0),
@@ -495,6 +496,7 @@ def create_datasource_property_table(cursor):
 (129, 'Observatory',    'RHESSI',           'RHESSI',          'RHESSI',          1),
 (130, 'Observatory',    'RHESSI',           'RHESSI',          'RHESSI',          1),
 (131, 'Observatory', 'PUNCH', 'PUNCH', 'PUNCH', 1),
+(134, 'Observatory', 'PUNCH', 'PUNCH', 'PUNCH', 1),
 (2000, 'Observatory', 'GOES', 'GOES-R', 'GOES-R', 1),
 (2001, 'Observatory', 'GOES', 'GOES-R', 'GOES-R', 1),
 (2002, 'Observatory', 'GOES', 'GOES-R', 'GOES-R', 1),
@@ -548,7 +550,8 @@ def create_datasource_property_table(cursor):
 (92, 'Instrument', 'SJI', 'SJI', 'Slit Jaw Imager', 2),
 (93, 'Instrument', 'SJI', 'SJI', 'Slit Jaw Imager', 2),
 (94, 'Instrument', 'GONG', 'GONG', 'GONG', 2),
-(131, 'Instrument', 'WFI+NFI', 'WFI+NFI', 'Wide and Near Field Imagers', 2),
+(131, 'Instrument', 'WFI+NFI', 'WFI+NFI', 'Clear low-noise science mosaic, bkg-sub & resolved into B & uncertainty layer', 2),
+(134, 'Instrument', 'WFI+NFI', 'WFI+NFI', 'Polarized low-noise science mosaic, bkg-sub & resolved into B, pB, & uncertainty layer', 2),
 (2000, 'Instrument', 'SUVI', 'SUVI', 'Solar UltraViolet Imager', 2),
 (2001, 'Instrument', 'SUVI', 'SUVI', 'Solar UltraViolet Imager', 2),
 (2002, 'Instrument', 'SUVI', 'SUVI', 'Solar UltraViolet Imager', 2),
@@ -574,6 +577,8 @@ def create_datasource_property_table(cursor):
 (86, 'Detector', 'HRI', 'HRI_EUV', 'High Resolution Imager Extreme Ultraviolet', 3),
 (87, 'Detector', 'HRI', 'HRI_LYA', 'High Resolution Imager Lyman-a', 3),
 (94, 'Detector', 'H-alpha', 'H-alpha', 'H-alpha', 3),
+(131, 'Measurement', 'Total Brightness', 'PUNCH Level-3 Intermediate F-corona Subtracted Unpolarized Mosaic', '', 3),
+(134, 'Measurement', 'Polarized Brightness', 'PUNCH Level-3 Intermediate F-corona Subtracted Polarized Mosaic', '', 3),
 (0, 'Measurement', '171', '171', '171 Ångström extreme ultraviolet', 3),
 (1, 'Measurement', '195', '195', '195 Ångström extreme ultraviolet', 3),
 (2, 'Measurement', '284', '284', '284 Ångström extreme ultraviolet', 3),
@@ -618,7 +623,6 @@ def create_datasource_property_table(cursor):
 (92, 'Measurement', '2832', '2832', '2832 Ångström', 3),
 (93, 'Measurement', '5000', '5000', '5000 Ångström', 3),
 (94, 'Measurement', '6562', '6562', 'H-alpha 6562 angstrom', 4),
-(131, 'Measurement', '530', '530', '', 3),
 (2000, 'Measurement', '94', '94', '94 Ångström',    3),
 (2001, 'Measurement', '131', '131', '131 Ångström', 3),
 (2002, 'Measurement', '171', '171', '171 Ångström', 3),

install/helioviewer/hvpull/net/daemon.py

@@ -539,8 +539,6 @@ def ingest(self, urls):
 
             # If everything looks good, move to archive and add to database
             # print image_params['date']
-            date_str = image_params['date'].strftime('%Y/%m/%d')
-
             # The files must be transcoded in order to work with JHelioviewer.
             # Therefore, any problem with the transcoding process must raise
             # an error.
@@ -566,13 +564,7 @@ def ingest(self, urls):
                 sys.exit(1)
 
             # Move to archive
-            if image_params['observatory'] == "Hinode":
-                directory = os.path.join(self.image_archive, image_params['nickname'], date_str, str(image_params['filter1']), str(image_params['filter2']))
-            elif image_params['observatory'] == "RHESSI":
-                directory = os.path.join(self.image_archive, image_params['nickname'], date_str, str(image_params['reconstruction_method']))
-            else:
-                directory = os.path.join(self.image_archive, image_params['nickname'], date_str, str(image_params['measurement']))
-
+            directory = os.path.join(self.image_archive, image_params['storage_path'])
             dest = os.path.join(directory, filename)
 
             image_params['filepath'] = dest

install/helioviewer/hvpull/servers/punch.py

@@ -1,18 +1,24 @@
 """PUNCH DataServer"""
 from helioviewer.hvpull.servers import DataServer
+from itertools import product,starmap
 import datetime
 import os
 
 class PUNCHDataServer(DataServer):
     def __init__(self):
         # TODO: Need to update to actual punch source when it's available.
-        DataServer.__init__(self, "/tmp/incoming/", "PUNCH")
+        DataServer.__init__(self, "https://umbra.nascom.nasa.gov/punch/L/3/", "PUNCH")
         self.pause = datetime.timedelta(minutes=30)
 
     def compute_directories(self, start_date, end_date):
         """Computes a list of remote directories expected to contain files"""
-        dirs = [os.path.join(self.uri)]
-        return dirs
+        top_level_folders = ["CAM","PAM"]
+
+        dirs = starmap(
+          lambda date, folder: os.path.join(self.uri, folder, date),
+          product(self.get_dates(start_date, end_date), top_level_folders)
+        )
+        return list(dirs)
 
     def get_datetime_from_file(self, filename):
         fname = os.path.basename(filename)

install/helioviewer/jp2.py

@@ -99,17 +99,8 @@ def insert_images(images, sources, rootdir, db, cursor, mysql, step_function=Non
         prev = ""
         source = sources
 
-        if img['observatory'] == "Hinode":
-            leafs = ["observatory", "instrument", "detector", "filter1", "filter2"]
-        elif img["observatory"] == "RHESSI":
-            leafs = ["observatory", "energy_band", "reconstruction_method"]
-        elif img["observatory"] == "PUNCH":
-            leafs = ["observatory", "instrument", "measurement"]
-        else:
-            leafs = ["observatory", "instrument", "detector", "measurement"]
-
+        leafs = JP2parser.get_detection_keys(img)
         for leaf in leafs:
-
             if img[leaf] != prev:
                 source = source[str(img[leaf])]
             prev = img[leaf]