Fixes for the getanc.py and modis_atteph.py login issues

[woodbri]

Hi all,I wrote two scripts to work-around the getanc.py and modis_atteph.py login issues. Offered here "as is" under public domain license in the hope this helps others. Please post fixes or improvements here for others to benefit.To install them move getanc.py to getanc-real.py and move modis_atteph.py to modis_atteph-real.py and then copy the appropriate script below into getanc.py or modis_atteph.py.the new script will call the realscript with '-d' option to get the filenames, then download those files and move them to where they belong.UPDATED: added support for ozone files in getanc.pyroot@B1B8B6F:/maps/nasa/tmp# cat modis_atteph.py#!/usr/bin/env python3import subprocessimport os.pathimport sysimport osdef fetchAndStore(fname):    (fpath, fn) = os.path.split(fname)    os.makedirs(fpath, exist_ok=True)    url = 'https://oceandata.sci.gsfc.nasa.gov (/cgi/getfile/) ' + fn    wget = ['wget', '-q', '--load-cookies', '~/.urs_cookies', '--save-cookies', '~/.urs_cookies', '--auth-no-challenge=on', '--keep-session-cookies', '--content-disposition', url]    out = subprocess.run(wget, stdout= subprocess.PIPE, stderr=subprocess.STDOUT)    if os.path.exists(fn):        mv = ['mv', fn, fname]        subprocess.run(mv)    elif os.path.exists(fn + '.bz2'):        mv = ['mv', fn + '.bz2', fname + '.bz2']        subprocess.run(mv)        unzip = ['bunzip2', fname + '.bz2']        subprocess.run(unzip)    else:        print('ERROR: failed to fetch: ' + url + "\n")        print(out.stdout.decode('utf-8'))atteph = '/u/oceandata/scripts/modis_atteph-real.py';cmd = [atteph, '-d'] + sys.argv[1:]out = subprocess.run(cmd, stdout= subprocess.PIPE, stderr=subprocess.STDOUT)lines = out.stdout.decode('utf-8').splitlines()for line in lines:    line = line.strip()    if "att1=" in line or "att2=" in line or "att3=" in line:        attfile = line[5:]        print(attfile + "\n")        if not os.path.exists(attfile):            fetchAndStore(attfile)    elif "eph1=" in line or "eph2=" in line or "eph3=" in line:        ephfile = line[5:]        print(ephfile + "\n")        if not os.path.exists(ephfile):            fetchAndStore(ephfile)root@B1B8B6F:/u/oceancolor-bin/scripts# cat getanc.py#!/usr/bin/env python3import subprocessimport os.pathimport sysimport osdef fetchAndStore(fname):    (fpath, fn) = os.path.split(fname)    os.makedirs(fpath, exist_ok=True)    url = 'https://oceandata.sci.gsfc.nasa.gov (/cgi/getfile/) ' + fn    wget = ['wget', '-q', '--load-cookies', '~/.urs_cookies', '--save-cookies', '~/.urs_cookies', '--auth-no-challenge=on', '--keep-session-cookies', '--content-disposition', url]    out = subprocess.run(wget, stdout= subprocess.PIPE, stderr=subprocess.STDOUT)    if os.path.exists(fn):        mv = ['mv', fn, fname]        subprocess.run(mv)    elif os.path.exists(fn + '.bz2'):        mv = ['mv', fn + '.bz2', fname + '.bz2']        subprocess.run(mv)        unzip = ['bunzip2', fname + '.bz2']        subprocess.run(unzip)    elif "ozone" in line:        ozfile = line[7:]        print(ozfile + "\n")        if not os.path.exists(ozfile):            fetchAndStore(ozfile)    else:        print('ERROR: failed to fetch: ' + url + "\n")        print(out.stdout.decode('utf-8'))getanc = '/u/oceandata/scripts/getanc-real.py';cmd = [getanc, '-d'] + sys.argv[1:]out = subprocess.run(cmd, stdout= subprocess.PIPE, stderr=subprocess.STDOUT)lines = out.stdout.decode('utf-8').splitlines()for line in lines:    line = line.strip()    if "icefile=" in line:        icefile = line[8:]        print(icefile + "\n")        if not os.path.exists(icefile):            fetchAndStore(icefile)    elif "met" in line:        metfile = line[5:]        print(metfile + "\n")        if not os.path.exists(metfile):            fetchAndStore(metfile)    elif "sstfile=" in line:        sstfile = line[8:]        print(sstfile + "\n")        if not os.path.exists(sstfile):            fetchAndStore(sstfile)

[pavel_babyak]

<...>
UPD: Sorry, wrong info   :confused:

[pavel_babyak]

You can make it easier.
Enough to patch ProcUtils.py to work with a .netrc

My solution:
patch

diff --git a/modules/ProcUtils.py b/modules/ProcUtils.py
index edceb41..547fe7b 100644
--- a/modules/ProcUtils.py
+++ b/modules/ProcUtils.py
@@ -6,7 +6,9 @@ SeaDAS library for commonly used functions within other python scripts
from __future__ import print_function

import sys
+import netrc

+netrc_dict=netrc.netrc()

#  ------------------ DANGER -------------------
#
@@ -98,6 +100,7 @@ def _httpdl(url, request, localpath='.', outputfilename=None, ntries=5,
         verbose - get chatty about connection issues (boolean, default False)
     """
     global ofile
+    global netrc_dict
     import os
     import re
     import socket
@@ -114,6 +117,13 @@ def _httpdl(url, request, localpath='.', outputfilename=None, ntries=5,
         os.umask(0o02)
         os.makedirs(localpath, mode=0o2775)

+    cred=netrc_dict.authenticators(url)
+    if cred is not None:
+        from base64 import b64encode
+        ( _user, _acc, _pass ) = cred
+        userAndPass = b64encode(bytes(_user + ':' + _pass, "utf-8"))
+        reqHeaders.update({ 'Authorization' : 'Basic %s' %  userAndPass })
+
     urlConn, proxy = httpinit(url, timeout=timeout, urlConn=urlConn)

     try:


[pavel_babyak]

Previous variant not working :(

Trying to use requests library, should works:


diff --git a/modules/ProcUtils.py b/modules/ProcUtils.py
index edceb41..6559339 100644
--- a/modules/ProcUtils.py
+++ b/modules/ProcUtils.py
@@ -6,7 +6,7 @@ SeaDAS library for commonly used functions within other python scripts
from __future__ import print_function

import sys
-
+import requests

#  ------------------ DANGER -------------------
#
@@ -251,9 +251,46 @@ def _httpdl(url, request, localpath='.', outputfilename=None, ntries=5,
def httpdl(url, request, localpath='.', outputfilename=None, ntries=5,
            uncompress=False, timeout=30., reqHeaders={}, verbose=False,
            reuseConn=False, urlConn=None, chunk_size=DEFAULT_CHUNK_SIZE):
-    urlConn, status = _httpdl(url, request, localpath, outputfilename, ntries,
-                              uncompress, timeout, reqHeaders, verbose,
-                              reuseConn, urlConn, chunk_size)
+
+    import requests
+    import os
+    import re
+
+    r = requests.get("https://%s%s" % (url,request), stream=True)
+
+    if r.status_code == 200 or r.status_code == 206:
+
+        if outputfilename:
+            ofile = os.path.join(localpath, outputfilename)
+        else:
+            if 'content-disposition' in r.headers:
+                ofile = os.path.join(localpath, r.headers.get('content-disposition').split('filename=')[1] )
+            else:
+                ofile = os.path.join(localpath, os.path.basename(request))
+
+        if r.status_code == 200:
+            f = open(ofile, 'wb')
+        else:
+            f = open(ofile, 'ab')
+
+        for chunk in r.iter_content(chunk_size=chunk_size):
+            if chunk: # filter out keep-alive new chunks
+                f.write(chunk)
+
+        f.flush()
+
+        if re.search(".(Z|gz|bz2)$", ofile) and uncompress:
+            compressStatus = uncompressFile(ofile)
+            if compressStatus:
+                status = compressStatus
+        else:
+            status = 0
+    else:
+        status = r.status_code
+
+#    urlConn, status = _httpdl(url, request, localpath, outputfilename, ntries,
+#                              uncompress, timeout, reqHeaders, verbose,
+#                              reuseConn, urlConn, chunk_size)
     if reuseConn:
         return (urlConn, status)
     else:


[ionutserban]

Hi Pavel,

I am not an expert in Ubuntu. Could you please briefly describe how do patch ProcUtils.py to work with a .netrc? What step should I follow, more exactly.

Thank you very much!

Best,
Ionut

[pavel_babyak]

Hi.
Patch in this thread have one bug.

I wrote a separate post with the latest version of the patch and described how to apply it.

[josl]

using `requests` is a million times better than some subprocess wget hack. Seems like switching to requests is what the maintainers implemented in the Seadas scripts.