1. tbrugz
  2. dados-tse

Commits

tbrugz  committed 5e2004c Draft

transf: bens-candidatos: lookup na tabela 'tse_tipos_bens' antes de insert em 'tse_candidatos_bens'

  • Participants
  • Parent commits a39f384
  • Branches default

Comments (0)

Files changed (3)

File TODO.md

View file
  • Ignore whitespace
 ? aceitar + de 1 arquivo csv por transformaусo
 x caminhos relativos para arquivos
 x mover info de conexсo de BD para kettle.properties
- 
+
 x modulo municipal (esquema multi-eleiусo)
 - otimizaушes mondrian-schema, banco de dados (indices?), ...
-- bens_cand: dimension lookup/update
+  - tabelas agregadas?
+x bens_cand: dimension (table!) lookup/update
+? bens_cand: candidato inexistente (erro FK). parece erro na origem dos dados (candidato (ainda) nao cadastrado?)
 
 
 tech/serviуos

File kettle/repositorio-de-dados-eleitorais/origem_CANDBEM.ktr

View file
  • Ignore whitespace
   </info>
   <notepads>
     <notepad>
-      <note>usar dimension lookup&#47;update?</note>
+      <note>usar dimension lookup&#47;update?
+
+FK tipo bem: ano_eleicao + cd_tipo_bem?</note>
       <xloc>299</xloc>
       <yloc>393</yloc>
-      <width>163</width>
-      <heigth>22</heigth>
+      <width>216</width>
+      <heigth>47</heigth>
       <fontname>Microsoft Sans Serif</fontname>
       <fontsize>8</fontsize>
       <fontbold>N</fontbold>
     </attributes>
   </connection>
   <order>
-  <hop> <from>renames</from><to>tipos bens</to><enabled>Y</enabled> </hop>  <hop> <from>tipos bens</from><to>Unique rows (HashSet)</to><enabled>Y</enabled> </hop>  <hop> <from>renames</from><to>i&#47;u - tse_candidatos_bens</to><enabled>N</enabled> </hop>  <hop> <from>Unique rows (HashSet)</from><to>i&#47;u - tipos bens</to><enabled>N</enabled> </hop>  <hop> <from>renames</from><to>tse_candidatos_bens</to><enabled>Y</enabled> </hop>  <hop> <from>tse_candidatos_bens</from><to>Erros bens_cand</to><enabled>Y</enabled> </hop>  <hop> <from>CSV - lista bens de candidatos</from><to>data&#47;hora invalida</to><enabled>Y</enabled> </hop>  <hop> <from>data&#47;hora invalida</from><to>renames</to><enabled>Y</enabled> </hop>  </order>
+  <hop> <from>tse_candidatos_bens</from><to>Erros bens_cand</to><enabled>Y</enabled> </hop>  <hop> <from>CSV - lista bens de candidatos</from><to>data&#47;hora invalida</to><enabled>Y</enabled> </hop>  <hop> <from>data&#47;hora invalida</from><to>renames</to><enabled>Y</enabled> </hop>  <hop> <from>renames</from><to>i&#47;u - tipos bens</to><enabled>Y</enabled> </hop>  <hop> <from>i&#47;u - tipos bens</from><to>tse_candidatos_bens</to><enabled>Y</enabled> </hop>  </order>
   <step>
     <name>CSV - lista bens de candidatos</name>
     <type>CsvInput</type>
     </step>
 
   <step>
-    <name>Unique rows (HashSet)</name>
-    <type>UniqueRowsByHashSet</type>
-    <description/>
-    <distribute>Y</distribute>
-    <copies>1</copies>
-         <partitioning>
-           <method>none</method>
-           <schema_name/>
-           </partitioning>
-      <store_values>N</store_values>
-      <reject_duplicate_row>N</reject_duplicate_row>
-      <error_description/>
-    <fields>      <field>        <name>CD_TIPO_BEM</name>
-        </field>      </fields>     <cluster_schema/>
- <remotesteps>   <input>   </input>   <output>   </output> </remotesteps>    <GUI>
-      <xloc>305</xloc>
-      <yloc>339</yloc>
-      <draw>Y</draw>
-      </GUI>
-    </step>
-
-  <step>
     <name>data&#47;hora invalida</name>
     <type>ScriptValueMod</type>
     <description/>
            </partitioning>
     <connection>dw_pgsql_tse</connection>
     <commit>100</commit>
-    <update_bypassed>N</update_bypassed>
+    <update_bypassed>Y</update_bypassed>
     <lookup>
       <schema/>
       <table>tse_tipos_bens</table>
     </step>
 
   <step>
-    <name>i&#47;u - tse_candidatos_bens</name>
-    <type>InsertUpdate</type>
-    <description/>
-    <distribute>Y</distribute>
-    <copies>1</copies>
-         <partitioning>
-           <method>none</method>
-           <schema_name/>
-           </partitioning>
-    <connection>dw_pgsql_tse</connection>
-    <commit>100</commit>
-    <update_bypassed>N</update_bypassed>
-    <lookup>
-      <schema/>
-      <table>tse_candidatos_bens</table>
-      <key>
-        <name>SEQ_CANDIDATO</name>
-        <field>SEQ_CANDIDATO</field>
-        <condition>=</condition>
-        <name2/>
-      </key>
-      <value>
-        <name>ANO_ELEICAO</name>
-        <rename>ANO_ELEICAO</rename>
-        <update>Y</update>
-      </value>
-      <value>
-        <name>SEQ_CANDIDATO</name>
-        <rename>SEQ_CANDIDATO</rename>
-        <update>N</update>
-      </value>
-      <value>
-        <name>CD_TIPO_BEM</name>
-        <rename>CD_TIPO_BEM</rename>
-        <update>Y</update>
-      </value>
-      <value>
-        <name>DS_TIPO_BEM</name>
-        <rename>DS_TIPO_BEM</rename>
-        <update>Y</update>
-      </value>
-      <value>
-        <name>DS_BEM</name>
-        <rename>DS_BEM</rename>
-        <update>Y</update>
-      </value>
-      <value>
-        <name>VL_BEM</name>
-        <rename>VL_BEM</rename>
-        <update>Y</update>
-      </value>
-      <value>
-        <name>DATA_ULTIMA_ATUALIZACAO</name>
-        <rename>DATA_ULTIMA_ATUALIZACAO</rename>
-        <update>Y</update>
-      </value>
-      <value>
-        <name>HORA_ULTIMA_ATUALIZACAO</name>
-        <rename>HORA_ULTIMA_ATUALIZACAO</rename>
-        <update>Y</update>
-      </value>
-    </lookup>
-     <cluster_schema/>
- <remotesteps>   <input>   </input>   <output>   </output> </remotesteps>    <GUI>
-      <xloc>318</xloc>
-      <yloc>207</yloc>
-      <draw>Y</draw>
-      </GUI>
-    </step>
-
-  <step>
     <name>renames</name>
     <type>SelectValues</type>
     <description/>
     </step>
 
   <step>
-    <name>tipos bens</name>
-    <type>SelectValues</type>
-    <description/>
-    <distribute>Y</distribute>
-    <copies>1</copies>
-         <partitioning>
-           <method>none</method>
-           <schema_name/>
-           </partitioning>
-    <fields>      <field>        <name>CD_TIPO_BEM</name>
-        <rename/>
-        <length>-2</length>
-        <precision>-2</precision>
-      </field>      <field>        <name>DS_TIPO_BEM</name>
-        <rename/>
-        <length>-2</length>
-        <precision>-2</precision>
-      </field>        <select_unspecified>N</select_unspecified>
-    </fields>     <cluster_schema/>
- <remotesteps>   <input>   </input>   <output>   </output> </remotesteps>    <GUI>
-      <xloc>188</xloc>
-      <yloc>338</yloc>
-      <draw>Y</draw>
-      </GUI>
-    </step>
-
-  <step>
     <name>tse_candidatos_bens</name>
     <type>TableOutput</type>
     <description/>

File modelo/saiku-tse-datasource.template.txt

View file
  • Ignore whitespace
-# Mondrian on Postgresql
+# Mondrian on Postgresql/H2
 # ----------------------------------------------
 type=OLAP
 name=tse
 driver=mondrian.olap4j.MondrianOlap4jDriver
+
+# postgresql
 #location=jdbc:mondrian:Jdbc=jdbc:postgresql://localhost/tse;Catalog=<absolute-project-dir>/modelo/sqldump-dados-tse/tse-mondrianschema.xml;JdbcDrivers=org.postgresql.Driver;
+# h2
 location=jdbc:mondrian:Jdbc=jdbc:h2:tcp://localhost/tse;Catalog=<absolute-project-dir>/modelo/sqldump-dados-tse/tse-mondrianschema.xml;JdbcDrivers=org.h2.Driver;
+
 username=postgres
 password=postgres