Mule stream design to handle large files from a remote location

I have a thread with the following steps:

1) Select file from SFTP

sever source

2) Copy it to local storage

3) Process file using copy in local storage

4) Put the processed file (to be converted) to the target server SFTP

5) Move the file present in the source SFTP

to a different folder on the original server SFTP

(I couldn't find a way to do this, and hence I copy from the temporary location back to the SFTP

processed folder)

It looks like a standard workflow, however I couldn't find any advice on how to specifically implement this in Mule.

Flow image

My current implementation is described below:

<file:connector name="tempFile" workDirectory="${temp.file.location}/work"
    workFileNamePattern="#[message.inboundProperties.originalFilename]"
    autoDelete="true" streaming="false" validateConnections="true"
    doc:name="File" />

<sftp:connector name="InputSFTP" validateConnections="true" keepFileOnError="true" doc:name="SFTP" >
    <reconnect frequency="${reconnectfrequency}" count="5"/>
</sftp:connector>

<sftp:connector name="DestinationSFTP" validateConnections="true" pollingFrequency="30000" doc:name="SFTP">
    <reconnect frequency="${reconnectfrequency}" count="5"/>
</sftp:connector>
<smtp:gmail-connector name="Gmail" contentType="text/plain" validateConnections="true" doc:name="Gmail"/>


<flow name="DownloadFTPFileIntoLocalFlow" processingStrategy="synchronous" tracking:enable-default-events="true">
    <sftp:inbound-endpoint connector-ref="InputSFTP" host="${source.host}" port="22" path="${source.path}" user="${source.username}" 
    password="${source.password}" responseTimeout="90000" pollingFrequency="120000" sizeCheckWaitTime="1000" doc:name="InputSFTP" autoDelete="true">
        <file:filename-regex-filter pattern="[Z].*\.csv" caseSensitive="false" />
    </sftp:inbound-endpoint>
    <file:outbound-endpoint path="${temp.file.location}" responseTimeout="10000" doc:name="Templocation" outputPattern="#[message.inboundProperties.originalFilename]" connector-ref="tempFile" />
    <exception-strategy ref="Default_Exception_Strategy" doc:name="Reference Exception Strategy"/>
</flow>
<flow name="ProcessCSVFlow" processingStrategy="synchronous" tracking:enable-default-events="true">
    <file:inbound-endpoint path="${temp.file.location}" connector-ref="tempFile" pollingFrequency="180000" fileAge="10000" responseTimeout="10000" doc:name="TempFileLocation"/>
    <transformer ref="enrichWithHeaderAndEndOfFileTransformer" doc:name="headerAndEOFEnricher" />
    <set-variable variableName="outputfilename" value="#['Mercury'+server.dateTime.year+server.dateTime.month+server.dateTime.dayOfMonth+server.dateTime.hours +server.dateTime.minutes+server.dateTime.seconds+'.csv']" doc:name="outputfilename"/>
    <sftp:outbound-endpoint exchange-pattern="one-way" connector-ref="DestinationSFTP" host="${destination.host}" port="22" responseTimeout="10000" doc:name="DestinationSFTP" 
    outputPattern="#[outputfilename]" path="${destination.path}" user="${destination.username}" password="${destination.password}"/>
    <gzip-compress-transformer/>
    <sftp:outbound-endpoint exchange-pattern="one-way" connector-ref="InputSFTP" host="${source.host}" port="22" responseTimeout="10000" doc:name="SourceArchiveSFTP" 
    outputPattern="#[outputfilename].gzip" path="Archive" user="${source.username}" password="${source.password}"/>
    <set-payload value="Hello world" doc:name="Set Payload"/>
    <smtp:outbound-endpoint host="${smtp.host}" port="${smtp.port}" user="${smtp.from.address}" password="${smtp.from.password}" 
                            to="${smtp.to.address}" from="${smtp.from.address}" subject="${mail.success.subject}" responseTimeout="10000" 
                            doc:name="SuccessEmail" connector-ref="Gmail"/>
    <logger message="Process completed successfully" level="INFO" doc:name="Logger"/>
    <tracking:transaction id="#[server.dateTime]"/>
    <exception-strategy ref="Default_Exception_Strategy" doc:name="Reference Exception Strategy"/>

</flow>
<catch-exception-strategy name="Default_Exception_Strategy">
    <logger message="Exception has occured Payload is #[payload] and Message is #[message]" level="ERROR" doc:name="Logger"/>
    <!-- <smtp:outbound-endpoint host="localhost" responseTimeout="10000" doc:name="Failure Email"/> -->
</catch-exception-strategy>

      

+3


source to share


1 answer


Have you tried enabling autoDelete = "true" on the SFTP connector to force delete?

Also, is it not possible to execute flow1: SFTP-in → transform → file out, flow2: file-in → SFTP-out?



NTN

0


source







All Articles