diff -ur v14/site_ut.py v15/site_ut.py --- v14/site_ut.py 2020-03-30 23:42:24.000000000 +0900 +++ v15/site_ut.py 2020-04-01 00:45:55.000000000 +0900 @@ -1,8 +1,11 @@ #!/usr/bin/env python +import os + import empty import base import yaml_ut +import cmd_ut import arg import dbg @@ -16,7 +19,10 @@ for fn in reversed( base.path_lst( yaml_fn, from__file__ ) ): d.update( yaml_ut.load_fn( fn, {} ) ) - d_url = d.pop( 'url', {} ) + props = ( 'url', 'lmt_size' ) + d_props = {} + for prop in props: + d_props[ prop ] = d.pop( prop, {} ) def get(k=''): if not k: @@ -28,11 +34,11 @@ s = d.get( k ) if ' ' in s: lst = s.split() - e = empty.new( host=lst[0], user=lst[1], passwd=lst[2], path='', url='' ) + e = empty.new( host=lst[0], user=lst[1], passwd=lst[2], path='' ) if len( lst ) > 3: e.path = lst[3] - if k in d_url: - e.url = d_url.get( k ) + for (prop, d_p) in d_props.items(): + setattr( e, prop, d_p.get( k, '' ) ) return e lst = s.split( '/' ) @@ -43,20 +49,109 @@ return None if rpath: e.path = base.add_path( e.path, rpath ) - if k in d_url: - e.url = d_url.get( k ) - elif e.url and rpath: + + for (prop, d_p) in d_props.items(): + if k in d_p: + setattr( e, prop, d_p.get( k ) ) + + d_url = d_props.get( 'url' ) + if k not in d_url and e.url and rpath: e.url = base.add_path( e.url, rpath ) + return e return empty.new( locals() ) + +call = lambda cmd, defval='' : cmd_ut.call( cmd, defval, b2s=True ) + +def to_num(s): + if type(s) != str: + return s + s = s.lower() + us = 'tgmk ' + for i in range( len( us ) - 1 ): + s = s.replace( us[i], '*1024' + us[i+1] ) + return eval( s ) + +def mk_dl_scr(url, path, lst, scr='dl.sh', tmp='/tmp/ttt'): + buf = [] + def add(s): + buf.append( s ) + + add( '#!/bin/bash' ) + add( 'rm -rf {} ; mkdir {}'.format( tmp, tmp ) ) + for name in lst: + add( '( cd {} ; wget -q {}/{}/{} ; echo {} )'.format( tmp, url, path, name, name ) ) + add( 'cat {}/* > {}'.format( tmp, path ) ) + add( 'rm -rf {}'.format( tmp ) ) + + s = to_str( buf, last=True ) + + dbg.out('scr {}'.format( path + '/' + scr )) + with open( path + '/' + scr, 'w' ) as f: + f.write( s ) + +def put(url, path, lmt_size, scr='dl.sh'): + if not lmt_size or os.path.getsize( path ) <= to_num( lmt_size ): + cmd = cmd_ut.cmd_py( 'ftp_ut' ) + ' ' + path + call( cmd ) + return + + (dir_, name) = os.path.split( path ) + tmp_dir = cmd_ut.mk_tmp_dir( path + '_' ) + + cmd = 'mv {} {}/ ; mkdir {}'.format( path, tmp_dir, path ) + call( cmd ) + + cmd = 'cat {}/{} | ( cd {} ; split -b {} - )'.format( tmp_dir, name, path, lmt_size ) + call( cmd ) + + cmd = 'ls {}'.format( path ) + lst = call( cmd ).strip().split('\n') + + #dbg.out('lst={}'.format(lst)) + + mk_dl_scr( url, path, lst, scr ) + + lst.append( scr ) + + paths = to_str( lst, pre=path+'/', delim=' ' ) + cmd = cmd_ut.cmd_py( 'ftp_ut' ) + ' ' + paths + dbg.out( 'cmd={}'.format( cmd ) ) + call( cmd ) + + cmd_ut.rm_rf( path ) + cmd = 'mv {}/{} {} ; rmdir {}'.format( tmp_dir, name, path, tmp_dir ) + dbg.out( 'cmd={}'.format( cmd ) ) + call( cmd ) + +def get(url, path, scr='dl.sh'): + cmd = 'wget -q {}/{}'.format( url, path ) + if call( cmd, 'err' ) != 'err': + return + + cmd = 'wget -q -O- {}/{}/{} | sh'.format( url, path, scr ) + call( cmd ) + if __name__ == "__main__": + site = new( 'ftp_ut.yaml' ) + cmds = ( 'put', 'get' ) a = arg.new() k = a.pop() - site = new( 'ftp_ut.yaml' ) - e = site.get(k) - if e: - s = yaml_ut.dump_no_flow( vars( e ) ) - dbg.out( s, '' ) + if k in cmds: + st = site.get() # default + path = a.pop() + if path: + if k == 'put': + put( st.url, path, st.lmt_size ) + elif k == 'get': + get( st.url, path ) + else: + st = site.get( k ) + if st: + s = yaml_ut.dump_no_flow( vars( st ) ) + if type( s ) == bytes: + s = s.decode() + dbg.out( s, '' ) # EOF