Changeset 3430

Timestamp:

09/21/2009 12:54:31 PM (6 months ago)

Author:

brian

Message:

Upgrade of cmsSync to only use the PhEDEx datasvc.

Files:

• cmsSync/setup.py (modified) (1 diff)
• cmsSync/src/cmsSync/sync_file.py (added)
• cmsSync/src/cmsSync/sync_phedex.py (modified) (6 diffs)

cmsSync/setup.py

@@ -4 +4 @@
-setup( \
-    name="cmsSync",
-0
+1
-    author = "Brian Bockelman",
-    author_email="bbockelm@math.unl.edu",

cmsSync/src/cmsSync/sync_phedex.py

@@ -12 +12 @@
-from xml.dom.minidom import parse
-
+import sync_file
-from dCachePnfs import PnfsSizeWalker
-from cms_tools import pretty_size
@@ -32 +33 @@
-    parser.add_option('-z', '--size', action='store_true', dest="show_size",
-        default=False, help="Show the total filesystem size.")
+    parser.add_option("-i", "--inspect", dest="inspect", default=False,
+        action="store_true", help="Check the files with CMSSW")
+    parser.add_option("-c", "--cmsset", dest="cmsset", default=None,
+        help="Location of cmsset_default.sh")
+    parser.add_option("-v", "--cmsswVersion", dest="cmssw",
+        default="CMSSW_3_1_1", help="CMSSW version to use.")
+    parser.add_option("-a", "--accessProtocol", dest="access",
+        default="direct", help="Access protocol to check the files.")
-    options, args = parser.parse_args()
-    if len(args) == 0:
@@ -41 +50 @@
-def get_phedex_name(se_name):
-    url = os.path.join(nebraska_datasvc, 'map_se', se_name)
+    try:
+        fp = urllib2.urlopen(url)
+    except:
+        print url
+        raise
+    return fp.read()
+
+def get_phedex_name(se_name):
+    url = os.path.join(phedex_datasvc, 'json', 'prod', 'nodes')
-    fp = urllib2.urlopen(url)
-
+
+
+
+
+
+
-
-def get_tfc(se_name):
@@ -55 +78 @@
-    tfc = get_tfc(se_name)
-    phedex_name = get_phedex_name(se_name)
-
-
+
+
+
+
-    dom = parse(fp)
-    tfc = TfcMapper(dom)
@@ -126 +151 @@
-        do_lookup(query)
-
-2
+1
-    t2 = -time.time()
-    splits = splitBlocks(blocks, splitNum=numThreads)
@@ -260 +285 @@
-        pretty_size(extra_lfns_size, PrettyOptions))
-
+    if options.inspect:
+        print "About to start opening files with PyROOT to validate them." \
+            "  This may take awhile..."
+        check_lfns = reg_lfns.intersection(lfns)
+        check_pfns = [tfc.map_to_pfn(lfn, protocol=options.access) for \
+            lfn in check_lfns]
+        open("checked_files.txt", "w").write("\n".join(check_pfns))
+        print "Saved the PFNs we'll check to checked_files.txt."
+        if options.cmsset and not os.path.exists(options.cmsset):
+            print "Error!  Unable to find cmsset_default.sh; value given was:" \
+                " %s." % str(options.cmsset)
+            return 1
+        sync_file.bootstrap_cms(options.cmsset, options.cmssw)
+        bad_files = sync_file.verify_files(check_pfns)
+        open("bad_files.txt", "w").write("\n".join(bad_files))
+        print "There were %i files which failed to open in PyROOT.  These are" \
+            " written to bad_files.txt."
+
-if __name__ == '__main__':
-
-
+
+
+
+